update crawler

ox/web/amazon.py

@@ -7,7 +7,7 @@ from six.moves.urllib.parse import quote
 from ox import find_re, strip_tags, decode_html
 from ox.cache import read_url
 
-import lxml
+import lxml.html
 
 
 def findISBN(title, author):

ox/web/thepiratebay.py

@@ -25,7 +25,7 @@ def find_movies(query=None, imdb=None, max_results=10):
     if imdb:
         query = "tt" + normalize_imdbid(imdb)
     results = []
-    next = ["http://thepiratebay.org/search/%s/0/3/200" % quote(query), ]
+    next = ["https://thepiratebay.se/search/%s/0/3/200" % quote(query), ]
     page_count = 1
     while next and page_count < 4:
         page_count += 1
@@ -33,12 +33,12 @@ def find_movies(query=None, imdb=None, max_results=10):
         if not url.startswith('http'):
             if not url.startswith('/'):
                 url = "/" + url
-            url = "http://thepiratebay.org" + url
+            url = "https://thepiratebay.se" + url
         data = read_url(url, timeout=cache_timeout, unicode=True)
         regexp = '''<tr.*?<td class="vertTh"><a href="/browse/(.*?)".*?<td><a href="(/torrent/.*?)" class="detLink".*?>(.*?)</a>.*?</tr>'''
         for row in  re.compile(regexp, re.DOTALL).findall(data):
             torrentType = row[0]
-            torrentLink = "http://thepiratebay.org" + row[1]
+            torrentLink = "https://thepiratebay.se" + row[1]
             torrentTitle = decode_html(row[2])
             # 201 = Movies , 202 = Movie DVDR, 205 TV Shows
             if torrentType in ['201']:
@@ -61,7 +61,7 @@ def get_id(piratebayId):
 
 def exists(piratebayId):
     piratebayId = get_id(piratebayId)
-    return ox.net.exists("http://thepiratebay.org/torrent/%s" % piratebayId)
+    return ox.net.exists("https://thepiratebay.se/torrent/%s" % piratebayId)
 
 def get_data(piratebayId):
     _key_map = {
@@ -75,7 +75,7 @@ def get_data(piratebayId):
     torrent = dict()
     torrent[u'id'] = piratebayId
     torrent[u'domain'] = 'thepiratebay.org'
-    torrent[u'comment_link'] = 'http://thepiratebay.org/torrent/%s' % piratebayId
+    torrent[u'comment_link'] = 'https://thepiratebay.se/torrent/%s' % piratebayId
 
     data = read_url(torrent['comment_link'], unicode=True)
     torrent[u'title'] = find_re(data, '<title>(.*?) \(download torrent\) - TPB</title>')

ox/web/ubu.py

@@ -24,6 +24,12 @@ def get_data(url):
         'url': url,
         'type': re.compile('ubu.com/(.*?)/').findall(url)[0]
     }
+    if m['type'] == 'sound':
+        m['tracks'] = [{
+            'title': strip_tags(decode_html(t[1])).strip(),
+            'url': t[0]
+        } for t in re.compile('"(http.*?.mp3)"[^>]*>(.+)</a', re.IGNORECASE).findall(data)]
+    else:
         for videourl, title in re.compile('href="(http://ubumexico.centro.org.mx/.*?)">(.*?)</a>').findall(data):
             if videourl.endswith('.srt'):
                 m['srt'] = videourl
@@ -135,3 +141,12 @@ def get_ids():
                     ids.append(u)
     ids = [get_id(url) for url in list(set(ids))]
     return ids
+
+def get_sound_ids():
+    data = read_url('http://www.ubu.com/sound/')
+    ids = []
+    for url, author in re.compile('<a href="(\./.*?)">(.*?)</a>').findall(data):
+        url = 'http://www.ubu.com/sound' + url[1:]
+        ids.append(url)
+    ids = [get_id(url) for url in sorted(set(ids))]
+    return ids