0x2620/scrapeit
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use wikipedia search stupid

Browse source
This commit is contained in:
j 2007-09-16 19:00:22 +00:00
parent 9616b3477e
commit ee442c4090
1 changed files with 10 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
scrapeit/wikipedia.py
View file

@ -10,29 +10,14 @@ from imdb import IMDb
from google import google from google import google
def searchByImdb(imdb_id, title=None, director=None): def searchByImdb(imdb_id):
if not title: if len(imdb_id) != 7: return ''
i = IMDb(imdb_id) url = "http://en.wikipedia.org/w/index.php?title=Special%3ASearch&search=imdb_id%20" + imdb_id + "&fulltext=Search"
title = i.parseTitle() data = read_url(url)
director = i.parseCredits()['director'] soup = BeautifulSoup(data)
if director: result = soup('li', {"style":"padding-bottom: 1em;"})
director = director[0] if result:
else: url = result[0]('a')[0]['href']
director = '' url = "http://en.wikipedia.org%s" % url
q = u'%s %s site:en.wikipedia.org' % (title, director) return url
#print q.encode('utf-8')
for g in google(q, 7):
url = g[1]
data = read_url(url)
soup = BeautifulSoup(data)
edit_url = soup('a', dict(href=re.compile('action=edit'),
title=re.compile('You can edit this page.')))
if edit_url:
edit_url = edit_url[0]['href']
edit_url = "http://en.wikipedia.org%s" % edit_url.replace('&', '&')
data = read_url(edit_url)
w_imdb_id = data.find('imdb_id')
if w_imdb_id > 0:
if imdb_id in data[w_imdb_id:w_imdb_id+50]:
return url
return '' return ''