0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

make sure tests work again, fix to32

Browse source
This commit is contained in:
j 2010-09-03 23:19:19 +02:00
parent 269086d349
commit e4e6798433
8 changed files with 109 additions and 46 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
README
View file

@ -1,4 +1,4 @@
python-ox some tools to build tools python-ox - the web in a dict
Depends: Depends:
python2.5 python2.5

70
ox/format.py
View file

@ -2,21 +2,75 @@
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import math import math
import re import re
import string
def to32(q): def to32(q):
""" """
Converts an integer to base 32 Converts an integer to base 32
We exclude 4 of the 26 letters: I L O U. We exclude 4 of the 26 letters: I L O U.
http://www.crockford.com/wrmg/base32.html http://www.crockford.com/wrmg/base32.html
>>> to32(0)
'0'
>>> to32(10)
'A'
>>> to32(11)
'B'
>>> to32(12)
'C'
>>> to32(13)
'D'
>>> to32(14)
'E'
>>> to32(15)
'F'
>>> to32(16)
'G'
>>> to32(17)
'H'
>>> to32(18)
'J'
>>> to32(19)
'K'
>>> to32(20)
'M'
>>> to32(21)
'N'
>>> to32(22)
'P'
>>> to32(23)
'Q'
>>> to32(24)
'R'
>>> to32(25)
'S'
>>> to32(26)
'T'
>>> to32(27)
'V'
>>> to32(28)
'W'
>>> to32(29)
'X'
>>> to32(30)
'Y'
>>> to32(31)
'Z'
>>> to32(32)
'10'
>>> to32(33)
'11'
>>> to32(34)
'12'
>>> to32(35) >>> to32(35)
'13' '13'
>>> to32(119292) >>> to32(119292)
'3MgV' '3MfW'
>>> to32(939387374) >>> to32(939387374)
'wZwTgD' 'vZvTfE'
>>> to32(0) >>> from32(to32(939387374))
'0' 939387374
>>> to32(-393) >>> to32(-393)
Traceback (most recent call last): Traceback (most recent call last):
... ...
@ -24,7 +78,7 @@ def to32(q):
""" """
if q < 0: raise ValueError, "must supply a positive integer" if q < 0: raise ValueError, "must supply a positive integer"
letters = "0123456789ACBEDGFHKJMNQPSRTWVYXZ" letters = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
converted = [] converted = []
upper = True upper = True
while q != 0: while q != 0:
@ -44,9 +98,9 @@ def from32(q):
We exclude 4 of the 26 letters: I L O U. We exclude 4 of the 26 letters: I L O U.
http://www.crockford.com/wrmg/base32.html http://www.crockford.com/wrmg/base32.html
>>> form32(to32(35)) >>> from32(to32(35))
35 35
>>> form32(to32(119292)) >>> from32(to32(119292))
119292 119292
>>> from32(to32(0)) >>> from32(to32(0))
0 0
@ -88,7 +142,7 @@ def from32(q):
'Y': 30, 'Y': 30,
'Z': 31, 'Z': 31,
} }
base32 = "0123456789ACBEDGFHKJMNQPSRTWVYXZ" base32 = ('0123456789' +string.ascii_uppercase)[:32]
q = q.replace('-','') q = q.replace('-','')
q = ''.join([base32[_32map[i.upper()]] for i in q]) q = ''.join([base32[_32map[i.upper()]] for i in q])
return int(q, 32) return int(q, 32)

8
ox/torrent/__init__.py
View file

@ -3,7 +3,7 @@
# GPL 2007-2009 # GPL 2007-2009
from threading import Event from threading import Event
import hashlib from hashlib import sha1
import os import os
from bencode import bencode, bdecode from bencode import bencode, bdecode
@ -21,7 +21,7 @@ def getInfoHash(torrentFile):
metainfo_file = open(torrentFile, 'rb') metainfo_file = open(torrentFile, 'rb')
metainfo = bdecode(metainfo_file.read()) metainfo = bdecode(metainfo_file.read())
info = metainfo['info'] info = metainfo['info']
return hashlib.sha1(bencode(info)).hexdigest() return sha1(bencode(info)).hexdigest()
def getTorrentInfoFromFile(torrentFile): def getTorrentInfoFromFile(torrentFile):
f = open(torrentFile, 'rb') f = open(torrentFile, 'rb')
@ -32,6 +32,8 @@ def getTorrentInfoFromFile(torrentFile):
return tinfo return tinfo
def getTorrentInfo(data): def getTorrentInfo(data):
from bencode import bencode
"Returns Torrent Info from torrent file" "Returns Torrent Info from torrent file"
tinfo = {} tinfo = {}
metainfo = bdecode(data) metainfo = bdecode(data)
@ -52,7 +54,7 @@ def getTorrentInfo(data):
if key != 'info': if key != 'info':
tinfo[key] = metainfo[key] tinfo[key] = metainfo[key]
tinfo['size'] = file_length tinfo['size'] = file_length
tinfo['hash'] = hashlib.sha1(bencode(info)).hexdigest() tinfo['hash'] = sha1(bencode(info)).hexdigest()
tinfo['announce'] = metainfo['announce'] tinfo['announce'] = metainfo['announce']
return tinfo return tinfo

9
ox/web/criterion.py
View file

@ -18,13 +18,13 @@ def getUrl(id):
def getData(id): def getData(id):
''' '''
>>> getData('1333')['imdbId'] >>> getData('1333')['imdbId']
'0060304' u'0060304'
>>> getData('236')['posters'][0] >>> getData('236')['posters'][0]
'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg' u'http://criterion_production.s3.amazonaws.com/release_images/1586/ThirdManReplace.jpg'
>>> getData('786')['posters'][0] >>> getData('786')['posters'][0]
'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg' u'http://criterion_production.s3.amazonaws.com/product_images/185/343_box_348x490.jpg'
''' '''
data = { data = {
"url": getUrl(id) "url": getUrl(id)
@ -50,7 +50,7 @@ def getData(id):
data["posters"] = [result] data["posters"] = [result]
else: else:
html_ = readUrlUnicode(result) html_ = readUrlUnicode(result)
result = findRe(html_, "<a href=\"http://www.criterion.com/films/%s\">(.*?)</a>" % id) result = findRe(html_, '<a href="http://www.criterion.com/films/%s.*?">(.*?)</a>' % id)
result = findRe(result, "src=\"(.*?)\"") result = findRe(result, "src=\"(.*?)\"")
data["posters"] = [result.replace("_w100", "")] data["posters"] = [result.replace("_w100", "")]
result = findRe(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"") result = findRe(html, "<img alt=\"Film Still\" height=\"252\" src=\"(.*?)\"")
@ -60,6 +60,7 @@ def getData(id):
else: else:
data["stills"] = [findRe(html, "\"thumbnailURL\", \"(.*?)\"")] data["stills"] = [findRe(html, "\"thumbnailURL\", \"(.*?)\"")]
data["trailers"] = [findRe(html, "\"videoURL\", \"(.*?)\"")] data["trailers"] = [findRe(html, "\"videoURL\", \"(.*?)\"")]
data['imdbId'] = imdb.getMovieId(data['title'], data['director'], data['year']) data['imdbId'] = imdb.getMovieId(data['title'], data['director'], data['year'])
return data return data

42
ox/web/dailymotion.py
View file

@ -1,22 +1,22 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
import re import re
from urllib import unquote from urllib import unquote
from ox.cache import readUrl from ox.cache import readUrl
def getVideoUrl(url): def getVideoUrl(url):
''' '''
>>> getVideoUrl('http://www.dailymotion.com/relevance/search/priere%2Bpour%2Brefuznik/video/x3opar_priere-pour-refuznik-1-jeanluc-goda_shortfilms').split('?key')[0] >>> getVideoUrl('http://www.dailymotion.com/relevance/search/priere%2Bpour%2Brefuznik/video/x3opar_priere-pour-refuznik-1-jeanluc-goda_shortfilms').split('?auth')[0]
'http://www.dailymotion.com/get/16/320x240/flv/6191379.flv' 'http://www.dailymotion.com/cdn/FLV-320x240/video/x3opar_priere-pour-refuznik-1-jean-luc-god_shortfilms.flv'
>>> getVideoUrl('http://www.dailymotion.com/relevance/search/priere%2Bpour%2Brefuznik/video/x3ou94_priere-pour-refuznik-2-jeanluc-goda_shortfilms').split('?key')[0] >>> getVideoUrl('http://www.dailymotion.com/relevance/search/priere%2Bpour%2Brefuznik/video/x3ou94_priere-pour-refuznik-2-jeanluc-goda_shortfilms').split('?auth')[0]
'http://www.dailymotion.com/get/15/320x240/flv/6197800.flv' 'http://www.dailymotion.com/cdn/FLV-320x240/video/x3ou94_priere-pour-refuznik-2-jean-luc-god_shortfilms.flv'
''' '''
data = readUrl(url) data = readUrl(url)
video = re.compile('''video", "(.*?)"''').findall(data) video = re.compile('''video", "(.*?)"''').findall(data)
for v in video: for v in video:
v = unquote(v).split('@@')[0] v = unquote(v).split('@@')[0]
return "http://www.dailymotion.com" + v return v
return '' return ''

16
ox/web/imdb.py
View file

@ -249,14 +249,20 @@ class ImdbCombined(Imdb):
def getMovieId(title, director='', year=''): def getMovieId(title, director='', year=''):
''' '''
>>> getMovieId('The Matrix') >>> getMovieId('The Matrix')
'0133093' u'0133093'
>>> getMovieId('2 or 3 Things I Know About Her', 'Jean-Luc Godard')
u'0060304'
>>> getMovieId('2 or 3 Things I Know About Her', 'Jean-Luc Godard', '1967')
u'0060304'
''' '''
if year:
title = "%s (%s)" % (title, year)
if director: if director:
query = 'site:imdb.com %s "%s"' % (director, title) query = 'site:imdb.com %s "%s" ' % (director, title)
else: else:
query = 'site:imdb.com "%s"' % title query = 'site:imdb.com "%s" ' % title
if year:
query += year
for (name, url, desc) in google.find(query, 5, timeout=-1): for (name, url, desc) in google.find(query, 5, timeout=-1):
if url.startswith('http://www.imdb.com/title/tt'): if url.startswith('http://www.imdb.com/title/tt'):
return url[28:35] return url[28:35]

6
ox/web/impawards.py
View file

@ -14,7 +14,7 @@ def getData(id):
u'0102926' u'0102926'
>>> getData('1991/silence_of_the_lambs')['posters'][0] >>> getData('1991/silence_of_the_lambs')['posters'][0]
u'http://www.impawards.com/1991/posters/silence_of_the_lambs_ver1_xlg.jpg' u'http://www.impawards.com/1991/posters/silence_of_the_lambs_ver1.jpg'
>>> getData('1991/silence_of_the_lambs')['url'] >>> getData('1991/silence_of_the_lambs')['url']
u'http://www.impawards.com/1991/silence_of_the_lambs_ver1.html' u'http://www.impawards.com/1991/silence_of_the_lambs_ver1.html'
@ -77,10 +77,10 @@ def getIdsByPage(page):
return set(ids) return set(ids)
def getUrl(id): def getUrl(id):
url = "http://www.impawards.com/%s.html" % id url = u"http://www.impawards.com/%s.html" % id
html = readUrlUnicode(url) html = readUrlUnicode(url)
if findRe(html, "No Movie Posters on This Page"): if findRe(html, "No Movie Posters on This Page"):
url = "http://www.impawards.com/%s_ver1.html" % id url = u"http://www.impawards.com/%s_ver1.html" % id
return url return url
if __name__ == '__main__': if __name__ == '__main__':

2
setup.py
View file

@ -13,7 +13,7 @@ def get_bzr_version():
setup( setup(
name="ox", name="ox",
version="2.0.%s" % get_bzr_version() , version="2.0.%s" % get_bzr_version() ,
description="python-ox some tools to build tools", description="python-ox - the web in a dict",
author="0x", author="0x",
author_email="code@0x2620.org", author_email="code@0x2620.org",
url="http://code.0x2620.org/python-ox", url="http://code.0x2620.org/python-ox",