fix ddg results
This commit is contained in:
parent
8aef76e7cd
commit
454d53d68a
1 changed files with 12 additions and 4 deletions
|
@ -6,17 +6,25 @@ from six.moves import urllib
|
||||||
import ox
|
import ox
|
||||||
from ox import strip_tags, decode_html
|
from ox import strip_tags, decode_html
|
||||||
from ox.cache import read_url
|
from ox.cache import read_url
|
||||||
|
import lxml.html
|
||||||
|
|
||||||
|
|
||||||
def find(query, timeout=ox.cache.cache_timeout):
|
def find(query, timeout=ox.cache.cache_timeout):
|
||||||
|
"""
|
||||||
|
Returns tuples with title, url, description
|
||||||
|
"""
|
||||||
if not isinstance(query, bytes):
|
if not isinstance(query, bytes):
|
||||||
query = query.encode('utf-8')
|
query = query.encode('utf-8')
|
||||||
params = urllib.parse.urlencode({'q': query})
|
params = urllib.parse.urlencode({'q': query})
|
||||||
url = 'http://duckduckgo.com/html/?' + params
|
url = 'http://duckduckgo.com/html/?' + params
|
||||||
data = read_url(url, timeout=timeout).decode('utf-8')
|
data = read_url(url, timeout=timeout).decode('utf-8')
|
||||||
|
doc = lxml.html.document_fromstring(data)
|
||||||
results = []
|
results = []
|
||||||
regex = '<a .*?class="large" href="(.+?)">(.*?)</a>.*?<div class="snippet">(.*?)</div>'
|
for e in doc.xpath("//a[contains(@class, 'result__a')]"):
|
||||||
for r in re.compile(regex, re.DOTALL).findall(data):
|
url = e.attrib['href']
|
||||||
results.append((strip_tags(decode_html(r[1])), r[0], strip_tags(decode_html(r[2]))))
|
if 'uddg=' in url:
|
||||||
|
url = urllib.parse.unquote(url.split('&uddg=')[-1])
|
||||||
|
title = e.text_content()
|
||||||
|
description = ''
|
||||||
|
results.append((title, url, description))
|
||||||
return results
|
return results
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue