update Shared
This commit is contained in:
parent
e7ebbedd38
commit
6881f3471a
184 changed files with 13080 additions and 13691 deletions
|
|
@ -7,7 +7,7 @@ from six.moves.urllib.parse import quote
|
|||
from ox import find_re, strip_tags, decode_html
|
||||
from ox.cache import read_url
|
||||
|
||||
import lxml
|
||||
import lxml.html
|
||||
|
||||
|
||||
def findISBN(title, author):
|
||||
|
|
|
|||
|
|
@ -15,9 +15,14 @@ def get_data(id):
|
|||
details = cache.read_url('%s?output=json' % url)
|
||||
details = json.loads(details)
|
||||
for key in ('title', 'description', 'runtime'):
|
||||
data[key] = details['metadata'][key]
|
||||
if isinstance(data[key], list):
|
||||
data[key] = data[key][0]
|
||||
if key in details['metadata']:
|
||||
data[key] = details['metadata'][key]
|
||||
if isinstance(data[key], list):
|
||||
data[key] = data[key][0]
|
||||
if isinstance(data[key], basestring):
|
||||
data[key] = data[key].strip()
|
||||
if data[key][0] == '[' and data[key][-1] == ']':
|
||||
data[key] = data[key][1:-1]
|
||||
data['url'] = url
|
||||
data['image'] = 'http://archive.org/download/%s/format=thumbnail' % id
|
||||
data['ogg'] = 'http://archive.org/download/%s/format=Ogg+video' % id
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import re
|
|||
|
||||
import ox.cache
|
||||
from ox.cache import read_url
|
||||
from ox.html import strip_tags
|
||||
from ox.html import strip_tags, decode_html
|
||||
from ox.text import find_re
|
||||
|
||||
import imdb
|
||||
|
|
@ -36,14 +36,15 @@ def get_data(id, timeout=ox.cache.cache_timeout, get_imdb=False):
|
|||
html = ox.cache.read_url(data["url"], timeout=timeout)
|
||||
data["number"] = find_re(html, "<li>Spine #(\d+)")
|
||||
|
||||
data["title"] = find_re(html, "<h1 class=\"movietitle\">(.*?)</h1>")
|
||||
data["title"] = data["title"].split(u' \u2014 The Television Version')[0]
|
||||
data["title"] = decode_html(find_re(html, "<h1 class=\"movietitle\">(.*?)</h1>"))
|
||||
data["title"] = data["title"].split(u' \u2014 The Television Version')[0].strip()
|
||||
data["director"] = strip_tags(find_re(html, "<h2 class=\"director\">(.*?)</h2>"))
|
||||
results = find_re(html, '<div class="left_column">(.*?)</div>')
|
||||
results = re.compile("<li>(.*?)</li>").findall(results)
|
||||
data["country"] = results[0]
|
||||
data["year"] = results[1]
|
||||
data["synopsis"] = strip_tags(find_re(html, "<div class=\"content_block last\">.*?<p>(.*?)</p>"))
|
||||
data["synopsis"] = decode_html(strip_tags(find_re(html,
|
||||
"<div class=\"content_block last\">.*?<p>(.*?)</p>")))
|
||||
|
||||
result = find_re(html, "<div class=\"purchase\">(.*?)</div>")
|
||||
if 'Blu-Ray' in result or 'Essential Art House DVD' in result:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import re
|
|||
import time
|
||||
import unicodedata
|
||||
|
||||
from six.moves import urllib
|
||||
from six.moves.urllib.parse import urlencode
|
||||
from six import string_types
|
||||
|
||||
from .. import find_re, strip_tags, decode_html
|
||||
|
|
@ -37,7 +37,7 @@ class Imdb(SiteParser):
|
|||
'alternativeTitles': {
|
||||
'page': 'releaseinfo',
|
||||
're': [
|
||||
'name="akas".*?<table.*?>(.*?)</table>',
|
||||
'<table[^>]*?id="akas"[^>]*?>(.*?)</table>',
|
||||
"td>(.*?)</td>.*?<td>(.*?)</td>"
|
||||
],
|
||||
'type': 'list'
|
||||
|
|
@ -74,7 +74,7 @@ class Imdb(SiteParser):
|
|||
'type': 'list'
|
||||
},
|
||||
'connections': {
|
||||
'page': 'trivia?tab=mc',
|
||||
'page': 'movieconnections',
|
||||
're': '<h4 class="li_group">(.*?)</h4>(.*?)(<\/div>\n <a|<script)',
|
||||
'type': 'list'
|
||||
},
|
||||
|
|
@ -476,9 +476,8 @@ class Imdb(SiteParser):
|
|||
alt[title].append(c)
|
||||
self['alternativeTitles'] = []
|
||||
for t in sorted(alt, key=lambda a: sorted(alt[a])):
|
||||
if alt[t]:
|
||||
countries = sorted([normalize_country_name(c) or c for c in alt[t]])
|
||||
self['alternativeTitles'].append((t, countries))
|
||||
countries = sorted([normalize_country_name(c) or c for c in alt[t]])
|
||||
self['alternativeTitles'].append((t, countries))
|
||||
if not self['alternativeTitles']:
|
||||
del self['alternativeTitles']
|
||||
|
||||
|
|
@ -521,7 +520,7 @@ class Imdb(SiteParser):
|
|||
if len(description) == 2 and description[-1].strip() != '-':
|
||||
r['description'] = description[-1].strip()
|
||||
return r
|
||||
cc[rel] = list(map(get_conn, re.compile('<a href="/title/tt(\d{7})/">(.*?)</a>(.*?)<\/div', re.DOTALL).findall(data)))
|
||||
cc[rel] = list(map(get_conn, re.compile('<a href="/title/tt(\d{7})/?">(.*?)</a>(.*?)<\/div', re.DOTALL).findall(data)))
|
||||
|
||||
|
||||
self['connections'] = cc
|
||||
|
|
@ -665,7 +664,7 @@ def get_movie_by_title(title, timeout=-1):
|
|||
params['q'] = unicodedata.normalize('NFKC', params['q']).encode('latin-1')
|
||||
except:
|
||||
params['q'] = params['q'].encode('utf-8')
|
||||
params = urllib.urlencode(params)
|
||||
params = urlencode(params)
|
||||
url = "http://akas.imdb.com/find?" + params
|
||||
data = read_url(url, timeout=timeout, unicode=True)
|
||||
#if search results in redirect, get id of current page
|
||||
|
|
@ -741,7 +740,7 @@ def get_movie_id(title, director='', year='', timeout=-1):
|
|||
params['q'] = unicodedata.normalize('NFKC', params['q']).encode('latin-1')
|
||||
except:
|
||||
params['q'] = params['q'].encode('utf-8')
|
||||
params = urllib.urlencode(params)
|
||||
params = urlencode(params)
|
||||
url = "http://akas.imdb.com/find?" + params
|
||||
#print url
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# encoding: utf-8
|
||||
from __future__ import print_function
|
||||
import re
|
||||
from six.moves import urllib
|
||||
from six.moves.urllib.parse import urlencode
|
||||
|
||||
from ox.cache import read_url
|
||||
from ox.html import decode_html, strip_tags
|
||||
|
|
@ -29,7 +29,7 @@ def compose_url(request, parameters):
|
|||
if request == 'advancedSearch':
|
||||
url = 'http://ax.phobos.apple.com.edgesuite.net/WebObjects/MZSearch.woa/wa/advancedSearch?'
|
||||
if parameters['media'] == 'music':
|
||||
url += urllib.urlencode({
|
||||
url += urlencode({
|
||||
'albumTerm': parameters['title'],
|
||||
'allArtistNames': parameters['artist'],
|
||||
'composerTerm': '',
|
||||
|
|
@ -42,7 +42,7 @@ def compose_url(request, parameters):
|
|||
'songTerm': ''
|
||||
})
|
||||
elif parameters['media'] == 'movie':
|
||||
url += urllib.urlencode({
|
||||
url += urlencode({
|
||||
'actorTerm': '',
|
||||
'closedCaption': 0,
|
||||
'descriptionTerm': '',
|
||||
|
|
|
|||
|
|
@ -7,12 +7,6 @@ from ox import find_re, strip_tags
|
|||
|
||||
|
||||
def get_url(id=None, imdb=None):
|
||||
#this would also wor but does not cache:
|
||||
'''
|
||||
from urllib2 import urlopen
|
||||
u = urlopen(url)
|
||||
return u.url
|
||||
'''
|
||||
if imdb:
|
||||
url = "http://www.rottentomatoes.com/alias?type=imdbid&s=%s" % imdb
|
||||
data = read_url(url)
|
||||
|
|
|
|||
44
Shared/lib/python3.4/site-packages/ox/web/startpage.py
Normal file
44
Shared/lib/python3.4/site-packages/ox/web/startpage.py
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=4:sts=4:ts=4
|
||||
from six.moves import urllib
|
||||
|
||||
import lxml.html
|
||||
import ox
|
||||
|
||||
DEFAULT_MAX_RESULTS = 10
|
||||
DEFAULT_TIMEOUT = 24*60*60
|
||||
|
||||
def read_url(url, data=None, headers=ox.net.DEFAULT_HEADERS, timeout=DEFAULT_TIMEOUT):
|
||||
return ox.cache.read_url(url, data, headers, timeout, unicode=True)
|
||||
|
||||
def quote_plus(s):
|
||||
if not isinstance(s, bytes):
|
||||
s = s.encode('utf-8')
|
||||
return urllib.parse.quote_plus(s)
|
||||
|
||||
def find(query, max_results=DEFAULT_MAX_RESULTS, timeout=DEFAULT_TIMEOUT):
|
||||
"""
|
||||
Return max_results tuples with title, url, description
|
||||
|
||||
>>> find("The Matrix site:imdb.com", 1)[0][0]
|
||||
u'The Matrix (1999) - IMDb'
|
||||
|
||||
>>> find("The Matrix site:imdb.com", 1)[0][1]
|
||||
u'http://www.imdb.com/title/tt0133093/'
|
||||
"""
|
||||
results = []
|
||||
url = 'https://eu1.startpage.com/do/search?nosteeraway=1&abp=1&language=english&cmd=process_search&query=%s&x=0&y=0&cat=web&engine0=v1all' % quote_plus(query)
|
||||
|
||||
data = read_url(url, timeout=timeout)
|
||||
doc = lxml.html.document_fromstring(data)
|
||||
for r in doc.xpath("//div[contains(@class, 'result')]"):
|
||||
t = r.find('h3')
|
||||
if t is not None:
|
||||
title = t.text_content().strip()
|
||||
url = t.find('a').attrib['href']
|
||||
description = r.find_class('desc')[0].text_content()
|
||||
results.append((title, url, description))
|
||||
if len(results) >= max_results:
|
||||
break
|
||||
return results
|
||||
|
||||
|
|
@ -25,7 +25,7 @@ def find_movies(query=None, imdb=None, max_results=10):
|
|||
if imdb:
|
||||
query = "tt" + normalize_imdbid(imdb)
|
||||
results = []
|
||||
next = ["http://thepiratebay.org/search/%s/0/3/200" % quote(query), ]
|
||||
next = ["https://thepiratebay.se/search/%s/0/3/200" % quote(query), ]
|
||||
page_count = 1
|
||||
while next and page_count < 4:
|
||||
page_count += 1
|
||||
|
|
@ -33,12 +33,12 @@ def find_movies(query=None, imdb=None, max_results=10):
|
|||
if not url.startswith('http'):
|
||||
if not url.startswith('/'):
|
||||
url = "/" + url
|
||||
url = "http://thepiratebay.org" + url
|
||||
url = "https://thepiratebay.se" + url
|
||||
data = read_url(url, timeout=cache_timeout, unicode=True)
|
||||
regexp = '''<tr.*?<td class="vertTh"><a href="/browse/(.*?)".*?<td><a href="(/torrent/.*?)" class="detLink".*?>(.*?)</a>.*?</tr>'''
|
||||
for row in re.compile(regexp, re.DOTALL).findall(data):
|
||||
torrentType = row[0]
|
||||
torrentLink = "http://thepiratebay.org" + row[1]
|
||||
torrentLink = "https://thepiratebay.se" + row[1]
|
||||
torrentTitle = decode_html(row[2])
|
||||
# 201 = Movies , 202 = Movie DVDR, 205 TV Shows
|
||||
if torrentType in ['201']:
|
||||
|
|
@ -61,7 +61,7 @@ def get_id(piratebayId):
|
|||
|
||||
def exists(piratebayId):
|
||||
piratebayId = get_id(piratebayId)
|
||||
return ox.net.exists("http://thepiratebay.org/torrent/%s" % piratebayId)
|
||||
return ox.net.exists("https://thepiratebay.se/torrent/%s" % piratebayId)
|
||||
|
||||
def get_data(piratebayId):
|
||||
_key_map = {
|
||||
|
|
@ -75,7 +75,7 @@ def get_data(piratebayId):
|
|||
torrent = dict()
|
||||
torrent[u'id'] = piratebayId
|
||||
torrent[u'domain'] = 'thepiratebay.org'
|
||||
torrent[u'comment_link'] = 'http://thepiratebay.org/torrent/%s' % piratebayId
|
||||
torrent[u'comment_link'] = 'https://thepiratebay.se/torrent/%s' % piratebayId
|
||||
|
||||
data = read_url(torrent['comment_link'], unicode=True)
|
||||
torrent[u'title'] = find_re(data, '<title>(.*?) \(download torrent\) - TPB</title>')
|
||||
|
|
|
|||
|
|
@ -3,12 +3,14 @@
|
|||
from __future__ import print_function
|
||||
import re
|
||||
|
||||
from ox import find_re, strip_tags, decode_html
|
||||
import lxml.html
|
||||
|
||||
from ox import strip_tags, decode_html
|
||||
from ox.cache import read_url
|
||||
|
||||
|
||||
def get_id(url):
|
||||
return url.replace('http://www.ubu.com/', '').split('.html')[0]
|
||||
return url.replace('http://www.ubu.com/', '').split('.html')[0].replace('/./', '/')
|
||||
|
||||
def get_url(id):
|
||||
return 'http://www.ubu.com/%s.html' % id
|
||||
|
|
@ -22,51 +24,92 @@ def get_data(url):
|
|||
'url': url,
|
||||
'type': re.compile('ubu.com/(.*?)/').findall(url)[0]
|
||||
}
|
||||
for videourl, title in re.compile('<a href="(http://ubumexico.centro.org.mx/.*?)">(.*?)</a>').findall(data):
|
||||
if videourl.endswith('.srt'):
|
||||
m['srt'] = videourl
|
||||
elif not 'video' in m:
|
||||
m['video'] = videourl
|
||||
m['video'] = m['video'].replace('/video/ ', '/video/').replace(' ', '%20')
|
||||
if m['video'] == 'http://ubumexico.centro.org.mx/video/':
|
||||
del m['video']
|
||||
m['title'] = strip_tags(decode_html(title)).strip()
|
||||
if not 'url' in m:
|
||||
print(url, 'missing')
|
||||
if 'title' in m:
|
||||
m['title'] = re.sub('(.*?) \(\d{4}\)$', '\\1', m['title'])
|
||||
|
||||
match = re.compile("flashvars','file=(.*?.flv)'").findall(data)
|
||||
if match:
|
||||
m['flv'] = match[0]
|
||||
m['flv'] = m['flv'].replace('/video/ ', '/video/').replace(' ', '%20')
|
||||
|
||||
y = re.compile('\((\d{4})\)').findall(data)
|
||||
if y:
|
||||
m['year'] = int(y[0])
|
||||
d = re.compile('Director: (.+)').findall(data)
|
||||
if d:
|
||||
m['director'] = strip_tags(decode_html(d[0])).strip()
|
||||
|
||||
a = re.compile('<a href="(.*?)">Back to (.*?)</a>', re.DOTALL).findall(data)
|
||||
if a:
|
||||
m['artist'] = strip_tags(decode_html(a[0][1])).strip()
|
||||
if m['type'] == 'sound':
|
||||
m['tracks'] = [{
|
||||
'title': strip_tags(decode_html(t[1])).strip(),
|
||||
'url': t[0]
|
||||
} for t in re.compile('"(http.*?.mp3)"[^>]*>(.+)</a', re.IGNORECASE).findall(data)]
|
||||
else:
|
||||
a = re.compile('<a href="(.*?)">(.*?) in UbuWeb Film').findall(data)
|
||||
for videourl, title in re.compile('href="(http://ubumexico.centro.org.mx/.*?)">(.*?)</a>').findall(data):
|
||||
if videourl.endswith('.srt'):
|
||||
m['srt'] = videourl
|
||||
elif not 'video' in m:
|
||||
m['video'] = videourl
|
||||
m['video'] = m['video'].replace('/video/ ', '/video/').replace(' ', '%20')
|
||||
if m['video'] == 'http://ubumexico.centro.org.mx/video/':
|
||||
del m['video']
|
||||
if not 'title' in m:
|
||||
m['title'] = strip_tags(decode_html(title)).strip()
|
||||
if not 'url' in m:
|
||||
print(url, 'missing')
|
||||
if 'title' in m:
|
||||
m['title'] = re.sub('(.*?) \(\d{4}\)$', '\\1', m['title'])
|
||||
|
||||
if not 'title' in m:
|
||||
match = re.compile('<span id="ubuwork">(.*?)</span>').findall(data)
|
||||
if match:
|
||||
m['title'] = strip_tags(decode_html(match[0])).strip()
|
||||
if not 'title' in m:
|
||||
match = re.compile("<title>.*?&(.*?)</title>", re.DOTALL).findall(data)
|
||||
if match:
|
||||
m['title'] = re.sub('\s+', ' ', match[0]).strip()
|
||||
if ' - ' in m['title']:
|
||||
m['title'] = m['title'].split(' - ', 1)[-1]
|
||||
if 'title' in m:
|
||||
m['title'] = strip_tags(decode_html(m['title']).strip())
|
||||
match = re.compile("flashvars','file=(.*?.flv)'").findall(data)
|
||||
if match:
|
||||
m['flv'] = match[0]
|
||||
m['flv'] = m['flv'].replace('/video/ ', '/video/').replace(' ', '%20')
|
||||
|
||||
match = re.compile('''src=(.*?) type="video/mp4"''').findall(data)
|
||||
if match:
|
||||
m['mp4'] = match[0].strip('"').strip("'").replace(' ', '%20')
|
||||
if not m['mp4'].startswith('http'):
|
||||
m['mp4'] = 'http://ubumexico.centro.org.mx/video/' + m['mp4']
|
||||
elif 'video' in m and (m['video'].endswith('.mp4') or m['video'].endswith('.m4v')):
|
||||
m['mp4'] = m['video']
|
||||
|
||||
doc = lxml.html.document_fromstring(read_url(url))
|
||||
desc = doc.xpath("//div[contains(@id, 'ubudesc')]")
|
||||
if len(desc):
|
||||
txt = []
|
||||
for part in desc[0].text_content().split('\n\n'):
|
||||
if part == 'RESOURCES:':
|
||||
break
|
||||
if part.strip():
|
||||
txt.append(part)
|
||||
if txt:
|
||||
if len(txt) > 1 and txt[0].strip() == m.get('title'):
|
||||
txt = txt[1:]
|
||||
m['description'] = '\n\n'.join(txt).split('RESOURCES')[0].split('RELATED')[0].strip()
|
||||
y = re.compile('\((\d{4})\)').findall(data)
|
||||
if y:
|
||||
m['year'] = int(y[0])
|
||||
d = re.compile('Director: (.+)').findall(data)
|
||||
if d:
|
||||
m['director'] = strip_tags(decode_html(d[0])).strip()
|
||||
|
||||
a = re.compile('<a href="(.*?)">Back to (.*?)</a>', re.DOTALL).findall(data)
|
||||
if a:
|
||||
m['artist'] = strip_tags(decode_html(a[0][1])).strip()
|
||||
else:
|
||||
a = re.compile('<b>(.*?)\(b\..*?\d{4}\)').findall(data)
|
||||
a = re.compile('<a href="(.*?)">(.*?) in UbuWeb Film').findall(data)
|
||||
if a:
|
||||
m['artist'] = strip_tags(decode_html(a[0])).strip()
|
||||
elif m['id'] == 'film/lawder_color':
|
||||
m['artist'] = 'Standish Lawder'
|
||||
if 'artist' in m:
|
||||
m['artist'] = m['artist'].replace('in UbuWeb Film', '')
|
||||
m['artist'] = m['artist'].replace('on UbuWeb Film', '').strip()
|
||||
if m['id'] == 'film/coulibeuf':
|
||||
m['title'] = 'Balkan Baroque'
|
||||
m['year'] = 1999
|
||||
m['artist'] = strip_tags(decode_html(a[0][1])).strip()
|
||||
else:
|
||||
a = re.compile('<b>(.*?)\(b\..*?\d{4}\)').findall(data)
|
||||
if a:
|
||||
m['artist'] = strip_tags(decode_html(a[0])).strip()
|
||||
elif m['id'] == 'film/lawder_color':
|
||||
m['artist'] = 'Standish Lawder'
|
||||
|
||||
if 'artist' in m:
|
||||
m['artist'] = m['artist'].replace('in UbuWeb Film', '')
|
||||
m['artist'] = m['artist'].replace('on UbuWeb Film', '').strip()
|
||||
if m['id'] == 'film/coulibeuf':
|
||||
m['title'] = 'Balkan Baroque'
|
||||
m['year'] = 1999
|
||||
return m
|
||||
|
||||
def get_films():
|
||||
|
|
@ -98,3 +141,12 @@ def get_ids():
|
|||
ids.append(u)
|
||||
ids = [get_id(url) for url in list(set(ids))]
|
||||
return ids
|
||||
|
||||
def get_sound_ids():
|
||||
data = read_url('http://www.ubu.com/sound/')
|
||||
ids = []
|
||||
for url, author in re.compile('<a href="(\./.*?)">(.*?)</a>').findall(data):
|
||||
url = 'http://www.ubu.com/sound' + url[1:]
|
||||
ids.append(url)
|
||||
ids = [get_id(url) for url in sorted(set(ids))]
|
||||
return ids
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=4:sts=4:ts=4
|
||||
from urllib import quote, unquote_plus
|
||||
import urllib2
|
||||
import cookielib
|
||||
from six.moves.urllib.parse import quote, unquote_plus
|
||||
from six.moves import urllib
|
||||
from six.moves import http_cookiejar as cookielib
|
||||
import re
|
||||
from xml.dom.minidom import parseString
|
||||
import json
|
||||
|
|
@ -167,7 +167,7 @@ def download_webm(id, filename):
|
|||
stream_type = 'video/webm'
|
||||
url = "http://www.youtube.com/watch?v=%s" % id
|
||||
cj = cookielib.CookieJar()
|
||||
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
|
||||
opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
|
||||
opener.addheaders = [
|
||||
('User-Agent',
|
||||
'Mozilla/5.0 (X11; Linux i686; rv:2.0) Gecko/20100101 Firefox/4.0'),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue