inline bittorrent functions and fix some some issues

This commit is contained in:
j 2009-11-29 19:24:25 +01:00
parent 2786c701f0
commit 7c4de11d3b
6 changed files with 369 additions and 11 deletions

1
README
View file

@ -3,7 +3,6 @@ python-oxlib some tools to build tools
Depends: Depends:
python2.5 python2.5
python-chardet (http://chardet.feedparser.org/) python-chardet (http://chardet.feedparser.org/)
BitTornado(optional)
Usage: Usage:
import oxlib import oxlib

View file

@ -11,9 +11,5 @@ from text import *
import cache import cache
import net import net
#only works if BitTornado is installed
try:
from torrent import * from torrent import *
except:
pass

View file

@ -1,19 +1,18 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4 # vi:si:et:sw=4:sts=4:ts=4
# GPL 2007 # GPL 2007-2009
from threading import Event from threading import Event
import hashlib import hashlib
from os import stat
import os import os
from BitTornado.BT1.makemetafile import make_meta_file
from bencode import bencode, bdecode from bencode import bencode, bdecode
def createTorrent(file, url, params = {}, flag = Event(), def createTorrent(file, url, params = {}, flag = Event(),
progress = lambda x: None, progress_percent = 1): progress = lambda x: None, progress_percent = 1):
"Creates a torrent for a given file, using url as tracker url" "Creates a torrent for a given file, using url as tracker url"
from makemetafile import make_meta_file
return make_meta_file(file, url, params, flag, progress, progress_percent) return make_meta_file(file, url, params, flag, progress, progress_percent)
def getInfoHash(torrentFile): def getInfoHash(torrentFile):
@ -21,14 +20,14 @@ def getInfoHash(torrentFile):
metainfo_file = open(torrentFile, 'rb') metainfo_file = open(torrentFile, 'rb')
metainfo = bdecode(metainfo_file.read()) metainfo = bdecode(metainfo_file.read())
info = metainfo['info'] info = metainfo['info']
return hashlib.sha1(bencode(info)).hexdigest().upper() return hashlib.sha1(bencode(info)).hexdigest()
def getTorrentInfoFromFile(torrentFile): def getTorrentInfoFromFile(torrentFile):
f = open(torrentFile, 'rb') f = open(torrentFile, 'rb')
data = f.read() data = f.read()
f.close() f.close()
tinfo = getTorrentInfo(data) tinfo = getTorrentInfo(data)
tinfo['timestamp'] = stat(torrentFile).st_ctime tinfo['timestamp'] = os.stat(torrentFile).st_ctime
return tinfo return tinfo
def getTorrentInfo(data): def getTorrentInfo(data):

100
oxlib/torrent/btformats.py Normal file
View file

@ -0,0 +1,100 @@
# Written by Bram Cohen
# see LICENSE.txt for license information
from types import StringType, LongType, IntType, ListType, DictType
from re import compile
reg = compile(r'^[^/\\.~][^/\\]*$')
ints = (LongType, IntType)
def check_info(info):
if type(info) != DictType:
raise ValueError, 'bad metainfo - not a dictionary'
pieces = info.get('pieces')
if type(pieces) != StringType or len(pieces) % 20 != 0:
raise ValueError, 'bad metainfo - bad pieces key'
piecelength = info.get('piece length')
if type(piecelength) not in ints or piecelength <= 0:
raise ValueError, 'bad metainfo - illegal piece length'
name = info.get('name')
if type(name) != StringType:
raise ValueError, 'bad metainfo - bad name'
if not reg.match(name):
raise ValueError, 'name %s disallowed for security reasons' % name
if info.has_key('files') == info.has_key('length'):
raise ValueError, 'single/multiple file mix'
if info.has_key('length'):
length = info.get('length')
if type(length) not in ints or length < 0:
raise ValueError, 'bad metainfo - bad length'
else:
files = info.get('files')
if type(files) != ListType:
raise ValueError
for f in files:
if type(f) != DictType:
raise ValueError, 'bad metainfo - bad file value'
length = f.get('length')
if type(length) not in ints or length < 0:
raise ValueError, 'bad metainfo - bad length'
path = f.get('path')
if type(path) != ListType or path == []:
raise ValueError, 'bad metainfo - bad path'
for p in path:
if type(p) != StringType:
raise ValueError, 'bad metainfo - bad path dir'
if not reg.match(p):
raise ValueError, 'path %s disallowed for security reasons' % p
for i in xrange(len(files)):
for j in xrange(i):
if files[i]['path'] == files[j]['path']:
raise ValueError, 'bad metainfo - duplicate path'
def check_message(message):
if type(message) != DictType:
raise ValueError
check_info(message.get('info'))
if type(message.get('announce')) != StringType:
raise ValueError
def check_peers(message):
if type(message) != DictType:
raise ValueError
if message.has_key('failure reason'):
if type(message['failure reason']) != StringType:
raise ValueError
return
peers = message.get('peers')
if type(peers) == ListType:
for p in peers:
if type(p) != DictType:
raise ValueError
if type(p.get('ip')) != StringType:
raise ValueError
port = p.get('port')
if type(port) not in ints or p <= 0:
raise ValueError
if p.has_key('peer id'):
id = p['peer id']
if type(id) != StringType or len(id) != 20:
raise ValueError
elif type(peers) != StringType or len(peers) % 6 != 0:
raise ValueError
interval = message.get('interval', 1)
if type(interval) not in ints or interval <= 0:
raise ValueError
minint = message.get('min interval', 1)
if type(minint) not in ints or minint <= 0:
raise ValueError
if type(message.get('tracker id', '')) != StringType:
raise ValueError
npeers = message.get('num peers', 0)
if type(npeers) not in ints or npeers < 0:
raise ValueError
dpeers = message.get('done peers', 0)
if type(dpeers) not in ints or dpeers < 0:
raise ValueError
last = message.get('last', 0)
if type(last) not in ints or last < 0:
raise ValueError

View file

@ -0,0 +1,264 @@
# Written by Bram Cohen
# multitracker extensions by John Hoffman
# see LICENSE.txt for license information
from os.path import getsize, split, join, abspath, isdir
from os import listdir
from hashlib import sha1 as sha
from copy import copy
from string import strip
from bencode import bencode
from btformats import check_info
from threading import Event
from time import time
from traceback import print_exc
try:
from sys import getfilesystemencoding
ENCODING = getfilesystemencoding()
except:
from sys import getdefaultencoding
ENCODING = getdefaultencoding()
defaults = [
('announce_list', '',
'a list of announce URLs - explained below'),
('httpseeds', '',
'a list of http seed URLs - explained below'),
('piece_size_pow2', 0,
"which power of 2 to set the piece size to (0 = automatic)"),
('comment', '',
"optional human-readable comment to put in .torrent"),
('filesystem_encoding', '',
"optional specification for filesystem encoding " +
"(set automatically in recent Python versions)"),
('target', '',
"optional target file for the torrent")
]
default_piece_len_exp = 18
ignore = ['core', 'CVS']
def print_announcelist_details():
print (' announce_list = optional list of redundant/backup tracker URLs, in the format:')
print (' url[,url...][|url[,url...]...]')
print (' where URLs separated by commas are all tried first')
print (' before the next group of URLs separated by the pipe is checked.')
print (" If none is given, it is assumed you don't want one in the metafile.")
print (' If announce_list is given, clients which support it')
print (' will ignore the <announce> value.')
print (' Examples:')
print (' http://tracker1.com|http://tracker2.com|http://tracker3.com')
print (' (tries trackers 1-3 in order)')
print (' http://tracker1.com,http://tracker2.com,http://tracker3.com')
print (' (tries trackers 1-3 in a randomly selected order)')
print (' http://tracker1.com|http://backup1.com,http://backup2.com')
print (' (tries tracker 1 first, then tries between the 2 backups randomly)')
print ('')
print (' httpseeds = optional list of http-seed URLs, in the format:')
print (' url[|url...]')
def make_meta_file(file, url, params = {}, flag = Event(),
progress = lambda x: None, progress_percent = 1):
if params.has_key('piece_size_pow2'):
piece_len_exp = params['piece_size_pow2']
else:
piece_len_exp = default_piece_len_exp
if params.has_key('target') and params['target'] != '':
f = params['target']
else:
a, b = split(file)
if b == '':
f = a + '.torrent'
else:
f = join(a, b + '.torrent')
if piece_len_exp == 0: # automatic
size = calcsize(file)
if size > 8L*1024*1024*1024: # > 8 gig =
piece_len_exp = 21 # 2 meg pieces
elif size > 2*1024*1024*1024: # > 2 gig =
piece_len_exp = 20 # 1 meg pieces
elif size > 512*1024*1024: # > 512M =
piece_len_exp = 19 # 512K pieces
elif size > 64*1024*1024: # > 64M =
piece_len_exp = 18 # 256K pieces
elif size > 16*1024*1024: # > 16M =
piece_len_exp = 17 # 128K pieces
elif size > 4*1024*1024: # > 4M =
piece_len_exp = 16 # 64K pieces
else: # < 4M =
piece_len_exp = 15 # 32K pieces
piece_length = 2 ** piece_len_exp
encoding = None
if params.has_key('filesystem_encoding'):
encoding = params['filesystem_encoding']
if not encoding:
encoding = ENCODING
if not encoding:
encoding = 'ascii'
info = makeinfo(file, piece_length, encoding, flag, progress, progress_percent)
if flag.isSet():
return
check_info(info)
h = open(f, 'wb')
data = {'info': info, 'announce': strip(url), 'creation date': long(time())}
if params.has_key('comment') and params['comment']:
data['comment'] = params['comment']
if params.has_key('real_announce_list'): # shortcut for progs calling in from outside
data['announce-list'] = params['real_announce_list']
elif params.has_key('announce_list') and params['announce_list']:
l = []
for tier in params['announce_list'].split('|'):
l.append(tier.split(','))
data['announce-list'] = l
if params.has_key('real_httpseeds'): # shortcut for progs calling in from outside
data['httpseeds'] = params['real_httpseeds']
elif params.has_key('httpseeds') and params['httpseeds']:
data['httpseeds'] = params['httpseeds'].split('|')
h.write(bencode(data))
h.close()
def calcsize(file):
if not isdir(file):
return getsize(file)
total = 0L
for s in subfiles(abspath(file)):
total += getsize(s[1])
return total
def uniconvertl(l, e):
r = []
try:
for s in l:
r.append(uniconvert(s, e))
except UnicodeError:
raise UnicodeError('bad filename: '+join(*l))
return r
def uniconvert(s, e):
try:
if s.__class__.__name__ != 'unicode':
s = unicode(s,e)
except UnicodeError:
raise UnicodeError('bad filename: '+s)
return s.encode('utf-8')
def makeinfo(file, piece_length, encoding, flag, progress, progress_percent=1):
file = abspath(file)
if isdir(file):
subs = subfiles(file)
subs.sort()
pieces = []
sh = sha()
done = 0L
fs = []
totalsize = 0.0
totalhashed = 0L
for p, f in subs:
totalsize += getsize(f)
for p, f in subs:
pos = 0L
size = getsize(f)
fs.append({'length': size, 'path': uniconvertl(p, encoding)})
h = open(f, 'rb')
while pos < size:
a = min(size - pos, piece_length - done)
sh.update(h.read(a))
if flag.isSet():
return
done += a
pos += a
totalhashed += a
if done == piece_length:
pieces.append(sh.digest())
done = 0
sh = sha()
if progress_percent:
progress(totalhashed / totalsize)
else:
progress(a)
h.close()
if done > 0:
pieces.append(sh.digest())
return {'pieces': ''.join(pieces),
'piece length': piece_length, 'files': fs,
'name': uniconvert(split(file)[1], encoding) }
else:
size = getsize(file)
pieces = []
p = 0L
h = open(file, 'rb')
while p < size:
x = h.read(min(piece_length, size - p))
if flag.isSet():
return
pieces.append(sha(x).digest())
p += piece_length
if p > size:
p = size
if progress_percent:
progress(float(p) / size)
else:
progress(min(piece_length, size - p))
h.close()
return {'pieces': ''.join(pieces),
'piece length': piece_length, 'length': size,
'name': uniconvert(split(file)[1], encoding) }
def subfiles(d):
r = []
stack = [([], d)]
while len(stack) > 0:
p, n = stack.pop()
if isdir(n):
for s in listdir(n):
if s not in ignore and s[:1] != '.':
stack.append((copy(p) + [s], join(n, s)))
else:
r.append((p, n))
return r
def completedir(dir, url, params = {}, flag = Event(),
vc = lambda x: None, fc = lambda x: None):
files = listdir(dir)
files.sort()
ext = '.torrent'
if params.has_key('target'):
target = params['target']
else:
target = ''
togen = []
for f in files:
if f[-len(ext):] != ext and (f + ext) not in files:
togen.append(join(dir, f))
total = 0
for i in togen:
total += calcsize(i)
subtotal = [0]
def callback(x, subtotal = subtotal, total = total, vc = vc):
subtotal[0] += x
vc(float(subtotal[0]) / total)
for i in togen:
fc(i)
try:
t = split(i)[-1]
if t not in ignore and t[0] != '.':
if target != '':
params['target'] = join(target,t+ext)
make_meta_file(i, url, params, flag, progress = callback, progress_percent = 0)
except ValueError:
print_exc()