0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

update crawler

Browse source
This commit is contained in:
j 2015-11-03 23:16:34 +01:00
parent 4a8717ee76
commit 3ed213d6d7
3 changed files with 95 additions and 80 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
ox/web/amazon.py
View file

@ -7,7 +7,7 @@ from six.moves.urllib.parse import quote
from ox import find_re, strip_tags, decode_html from ox import find_re, strip_tags, decode_html
from ox.cache import read_url from ox.cache import read_url
import lxml import lxml.html
def findISBN(title, author): def findISBN(title, author):

10
ox/web/thepiratebay.py
View file

@ -25,7 +25,7 @@ def find_movies(query=None, imdb=None, max_results=10):
if imdb: if imdb:
query = "tt" + normalize_imdbid(imdb) query = "tt" + normalize_imdbid(imdb)
results = [] results = []
next = ["http://thepiratebay.org/search/%s/0/3/200" % quote(query), ] next = ["https://thepiratebay.se/search/%s/0/3/200" % quote(query), ]
page_count = 1 page_count = 1
while next and page_count < 4: while next and page_count < 4:
page_count += 1 page_count += 1
@ -33,12 +33,12 @@ def find_movies(query=None, imdb=None, max_results=10):
if not url.startswith('http'): if not url.startswith('http'):
if not url.startswith('/'): if not url.startswith('/'):
url = "/" + url url = "/" + url
url = "http://thepiratebay.org" + url url = "https://thepiratebay.se" + url
data = read_url(url, timeout=cache_timeout, unicode=True) data = read_url(url, timeout=cache_timeout, unicode=True)
regexp = '''<tr.*?<td class="vertTh"><a href="/browse/(.*?)".*?<td><a href="(/torrent/.*?)" class="detLink".*?>(.*?)</a>.*?</tr>''' regexp = '''<tr.*?<td class="vertTh"><a href="/browse/(.*?)".*?<td><a href="(/torrent/.*?)" class="detLink".*?>(.*?)</a>.*?</tr>'''
for row in re.compile(regexp, re.DOTALL).findall(data): for row in re.compile(regexp, re.DOTALL).findall(data):
torrentType = row[0] torrentType = row[0]
torrentLink = "http://thepiratebay.org" + row[1] torrentLink = "https://thepiratebay.se" + row[1]
torrentTitle = decode_html(row[2]) torrentTitle = decode_html(row[2])
# 201 = Movies , 202 = Movie DVDR, 205 TV Shows # 201 = Movies , 202 = Movie DVDR, 205 TV Shows
if torrentType in ['201']: if torrentType in ['201']:
@ -61,7 +61,7 @@ def get_id(piratebayId):
def exists(piratebayId): def exists(piratebayId):
piratebayId = get_id(piratebayId) piratebayId = get_id(piratebayId)
return ox.net.exists("http://thepiratebay.org/torrent/%s" % piratebayId) return ox.net.exists("https://thepiratebay.se/torrent/%s" % piratebayId)
def get_data(piratebayId): def get_data(piratebayId):
_key_map = { _key_map = {
@ -75,7 +75,7 @@ def get_data(piratebayId):
torrent = dict() torrent = dict()
torrent[u'id'] = piratebayId torrent[u'id'] = piratebayId
torrent[u'domain'] = 'thepiratebay.org' torrent[u'domain'] = 'thepiratebay.org'
torrent[u'comment_link'] = 'http://thepiratebay.org/torrent/%s' % piratebayId torrent[u'comment_link'] = 'https://thepiratebay.se/torrent/%s' % piratebayId
data = read_url(torrent['comment_link'], unicode=True) data = read_url(torrent['comment_link'], unicode=True)
torrent[u'title'] = find_re(data, '<title>(.*?) \(download torrent\) - TPB</title>') torrent[u'title'] = find_re(data, '<title>(.*?) \(download torrent\) - TPB</title>')

15
ox/web/ubu.py
View file

@ -24,6 +24,12 @@ def get_data(url):
'url': url, 'url': url,
'type': re.compile('ubu.com/(.*?)/').findall(url)[0] 'type': re.compile('ubu.com/(.*?)/').findall(url)[0]
} }
if m['type'] == 'sound':
m['tracks'] = [{
'title': strip_tags(decode_html(t[1])).strip(),
'url': t[0]
} for t in re.compile('"(http.*?.mp3)"[^>]*>(.+)</a', re.IGNORECASE).findall(data)]
else:
for videourl, title in re.compile('href="(http://ubumexico.centro.org.mx/.*?)">(.*?)</a>').findall(data): for videourl, title in re.compile('href="(http://ubumexico.centro.org.mx/.*?)">(.*?)</a>').findall(data):
if videourl.endswith('.srt'): if videourl.endswith('.srt'):
m['srt'] = videourl m['srt'] = videourl
@ -135,3 +141,12 @@ def get_ids():
ids.append(u) ids.append(u)
ids = [get_id(url) for url in list(set(ids))] ids = [get_id(url) for url in list(set(ids))]
return ids return ids
def get_sound_ids():
data = read_url('http://www.ubu.com/sound/')
ids = []
for url, author in re.compile('<a href="(\./.*?)">(.*?)</a>').findall(data):
url = 'http://www.ubu.com/sound' + url[1:]
ids.append(url)
ids = [get_id(url) for url in sorted(set(ids))]
return ids