0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

fix ddg results

Browse source
This commit is contained in:
j 2017-07-26 13:18:10 +02:00
parent 8aef76e7cd
commit 454d53d68a
1 changed files with 12 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
ox/web/duckduckgo.py
View file

@ -6,17 +6,25 @@ from six.moves import urllib
import ox import ox
from ox import strip_tags, decode_html from ox import strip_tags, decode_html
from ox.cache import read_url from ox.cache import read_url
import lxml.html
def find(query, timeout=ox.cache.cache_timeout): def find(query, timeout=ox.cache.cache_timeout):
"""
Returns tuples with title, url, description
"""
if not isinstance(query, bytes): if not isinstance(query, bytes):
query = query.encode('utf-8') query = query.encode('utf-8')
params = urllib.parse.urlencode({'q': query}) params = urllib.parse.urlencode({'q': query})
url = 'http://duckduckgo.com/html/?' + params url = 'http://duckduckgo.com/html/?' + params
data = read_url(url, timeout=timeout).decode('utf-8') data = read_url(url, timeout=timeout).decode('utf-8')
doc = lxml.html.document_fromstring(data)
results = [] results = []
regex = '<a .*?class="large" href="(.+?)">(.*?)</a>.*?<div class="snippet">(.*?)</div>' for e in doc.xpath("//a[contains(@class, 'result__a')]"):
for r in re.compile(regex, re.DOTALL).findall(data): url = e.attrib['href']
results.append((strip_tags(decode_html(r[1])), r[0], strip_tags(decode_html(r[2])))) if 'uddg=' in url:
url = urllib.parse.unquote(url.split('&uddg=')[-1])
title = e.text_content()
description = ''
results.append((title, url, description))
return results return results