decentral1se/pandora
1
0
Fork 0
forked from 0x2620/pandora
Code Issues Pull requests Projects Releases Wiki Activity

flake8

Browse source
This commit is contained in:
j 2016-07-01 17:41:53 +02:00
parent 3166bf5beb
commit d780045306
6 changed files with 72 additions and 70 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
pandora/archive/admin.py
View file

@ -7,12 +7,12 @@ import models
class FileAdmin(admin.ModelAdmin): class FileAdmin(admin.ModelAdmin):
search_fields = ['path','oshash', 'video_codec'] search_fields = ['path', 'oshash', 'video_codec']
list_display = ['available', 'wanted', 'selected', '__unicode__', 'public_id'] list_display = ['available', 'wanted', 'selected', '__unicode__', 'public_id']
list_display_links = ('__unicode__', ) list_display_links = ('__unicode__', )
def public_id(self, obj): def public_id(self, obj):
return '%s'%(obj.item.public_id) return '%s' % (obj.item.public_id)
admin.site.register(models.File, FileAdmin) admin.site.register(models.File, FileAdmin)

2
pandora/archive/chunk.py
View file

@ -23,7 +23,7 @@ def save_chunk(obj, file, chunk, offset, name, done_cb=None):
else: else:
path = file.path path = file.path
size = file.size size = file.size
if offset == None: if offset is None:
offset = size offset = size
elif offset > size: elif offset > size:
return False, size return False, size

2
pandora/archive/extract.py
View file

@ -496,7 +496,7 @@ def average_color(prefix, start=0, end=0, mode='antialias'):
timeline = timeline.crop(box) timeline = timeline.crop(box)
p = np.asarray(timeline.convert('RGB'), dtype=np.float32) p = np.asarray(timeline.convert('RGB'), dtype=np.float32)
p = np.sum(p, axis=0) / height #average color per frame p = np.sum(p, axis=0) / height # average color per frame
pixels.append(p) pixels.append(p)
if end and frames >= end: if end and frames >= end:
break break

70
pandora/archive/models.py
View file

@ -24,7 +24,8 @@ from person.models import get_name_sort
from chunk import save_chunk from chunk import save_chunk
import extract import extract
def data_path(f, x): return f.get_path('data.bin') def data_path(f, x):
return f.get_path('data.bin')
class File(models.Model): class File(models.Model):
AV_INFO = ( AV_INFO = (
@ -46,12 +47,12 @@ class File(models.Model):
oshash = models.CharField(max_length=16, unique=True) oshash = models.CharField(max_length=16, unique=True)
item = models.ForeignKey("item.Item", related_name='files', null=True) item = models.ForeignKey("item.Item", related_name='files', null=True)
path = models.CharField(max_length=2048, default="") # canoncial path/file path = models.CharField(max_length=2048, default="") # canoncial path/file
sort_path = models.CharField(max_length=2048, default="") # sort name sort_path = models.CharField(max_length=2048, default="") # sort name
type = models.CharField(default="", max_length=255) type = models.CharField(default="", max_length=255)
#editable # editable
extension = models.CharField(default="", max_length=255, null=True) extension = models.CharField(default="", max_length=255, null=True)
language = models.CharField(default="", max_length=255, null=True) language = models.CharField(default="", max_length=255, null=True)
part = models.CharField(default="", max_length=255, null=True) part = models.CharField(default="", max_length=255, null=True)
@ -77,20 +78,20 @@ class File(models.Model):
bits_per_pixel = models.FloatField(default=-1) bits_per_pixel = models.FloatField(default=-1)
pixels = models.BigIntegerField(default=0) pixels = models.BigIntegerField(default=0)
#This is true if derivative is available or subtitles where uploaded # This is true if derivative is available or subtitles where uploaded
available = models.BooleanField(default = False) available = models.BooleanField(default=False)
selected = models.BooleanField(default = False) selected = models.BooleanField(default=False)
uploading = models.BooleanField(default = False) uploading = models.BooleanField(default=False)
queued = models.BooleanField(default = False) queued = models.BooleanField(default=False)
encoding = models.BooleanField(default = False) encoding = models.BooleanField(default=False)
wanted = models.BooleanField(default = False) wanted = models.BooleanField(default=False)
failed = models.BooleanField(default = False) failed = models.BooleanField(default=False)
is_audio = models.BooleanField(default=False) is_audio = models.BooleanField(default=False)
is_video = models.BooleanField(default=False) is_video = models.BooleanField(default=False)
is_subtitle = models.BooleanField(default=False) is_subtitle = models.BooleanField(default=False)
#upload and data handling # upload and data handling
data = models.FileField(null=True, blank=True, data = models.FileField(null=True, blank=True,
upload_to=data_path) upload_to=data_path)
@ -161,11 +162,11 @@ class File(models.Model):
data['isEpisode'] = True data['isEpisode'] = True
data['directorSort'] = [get_name_sort(n) for n in self.item.get('director', [])] data['directorSort'] = [get_name_sort(n) for n in self.item.get('director', [])]
data['isEpisode'] = 'isEpisode' in data \ data['isEpisode'] = 'isEpisode' in data \
or data.get('season') != None \ or data.get('season') is not None \
or data.get('episode') != None \ or data.get('episode') is not None \
or data.get('episodes') not in ([], None) \ or data.get('episodes') not in ([], None) \
or (data.get('seriesTitle') != None and data.get('episodeTitle') != None) or (data.get('seriesTitle') is not None and data.get('episodeTitle') is not None)
if data['isEpisode'] and data['seriesYear'] == None: if data['isEpisode'] and data['seriesYear'] is None:
data['seriesYear'] = data['year'] data['seriesYear'] = data['year']
data['type'] = 'unknown' data['type'] = 'unknown'
if 'extension' in data and data['extension']: if 'extension' in data and data['extension']:
@ -178,7 +179,7 @@ class File(models.Model):
return data return data
def normalize_path(self): def normalize_path(self):
#FIXME: always use format_path # FIXME: always use format_path
if settings.CONFIG['site']['folderdepth'] == 4: if settings.CONFIG['site']['folderdepth'] == 4:
return self.normalize_item_path() return self.normalize_item_path()
else: else:
@ -193,6 +194,7 @@ class File(models.Model):
files = [] files = []
volume = self.instances.all()[0].volume volume = self.instances.all()[0].volume
def add_file(f): def add_file(f):
instance = f.instances.all()[0] instance = f.instances.all()[0]
files.append(f.get_path_info()) files.append(f.get_path_info())
@ -214,11 +216,11 @@ class File(models.Model):
def update_info(self, info, user): def update_info(self, info, user):
if not self.info: if not self.info:
#populate name sort with director if unknown # populate name sort with director if unknown
if info.get('director') and info.get('directorSort'): if info.get('director') and info.get('directorSort'):
for name, sortname in zip(info['director'], info['directorSort']): for name, sortname in zip(info['director'], info['directorSort']):
get_name_sort(name, sortname) get_name_sort(name, sortname)
#add all files in one folder to same item # add all files in one folder to same item
if self.instances.all().count(): if self.instances.all().count():
if info.get('isEpisode'): if info.get('isEpisode'):
prefix = os.path.splitext(self.instances.all()[0].path)[0] prefix = os.path.splitext(self.instances.all()[0].path)[0]
@ -268,7 +270,7 @@ class File(models.Model):
self.available = self.data and True or False self.available = self.data and True or False
else: else:
self.available = not self.uploading and \ self.available = not self.uploading and \
self.streams.filter(source=None, available=True).count() > 0 self.streams.filter(source=None, available=True).count()
super(File, self).save(*args, **kwargs) super(File, self).save(*args, **kwargs)
if update_path: if update_path:
self.path = self.normalize_path() self.path = self.normalize_path()
@ -279,7 +281,7 @@ class File(models.Model):
return os.path.join('media', h[:2], h[2:4], h[4:6], h[6:], name) return os.path.join('media', h[:2], h[2:4], h[4:6], h[6:], name)
def contents(self): def contents(self):
if self.data != None: if self.data is not None:
self.data.seek(0) self.data.seek(0)
return self.data.read() return self.data.read()
return None return None
@ -293,7 +295,7 @@ class File(models.Model):
if key not in subtitles: if key not in subtitles:
subtitles.append(key) subtitles.append(key)
srt.append(s) srt.append(s)
#subtitles should not overlap # subtitles should not overlap
for i in range(1, len(srt)): for i in range(1, len(srt)):
if srt[i-1]['out'] > srt[i]['in']: if srt[i-1]['out'] > srt[i]['in']:
srt[i-1]['out'] = srt[i]['in'] srt[i-1]['out'] = srt[i]['in']
@ -335,8 +337,7 @@ class File(models.Model):
def save_chunk_stream(self, chunk, offset, resolution, format, done): def save_chunk_stream(self, chunk, offset, resolution, format, done):
if not self.available: if not self.available:
stream, created = Stream.objects.get_or_create( stream, created = Stream.objects.get_or_create(file=self, resolution=resolution, format=format)
file=self, resolution=resolution, format=format)
name = stream.path(stream.name()) name = stream.path(stream.name())
def done_cb(): def done_cb():
@ -436,7 +437,7 @@ class File(models.Model):
extract.frame_direct(filename, fr.frame.path, pos) extract.frame_direct(filename, fr.frame.path, pos)
if os.path.exists(fr.frame.path): if os.path.exists(fr.frame.path):
fr.save() fr.save()
os.chmod(fr.frame.path, 0644) os.chmod(fr.frame.path, 0o644)
self.item.select_frame() self.item.select_frame()
def extract_stream(self): def extract_stream(self):
@ -558,7 +559,7 @@ class Volume(models.Model):
name = models.CharField(max_length=1024) name = models.CharField(max_length=1024)
def __unicode__(self): def __unicode__(self):
return u"%s's %s"% (self.user, self.name) return u"%s's %s" % (self.user, self.name)
def json(self): def json(self):
return { return {
@ -589,7 +590,7 @@ class Instance(models.Model):
volume = models.ForeignKey(Volume, related_name='files') volume = models.ForeignKey(Volume, related_name='files')
def __unicode__(self): def __unicode__(self):
return u"%s's %s <%s>"% (self.volume.user, self.path, self.file.oshash) return u"%s's %s <%s>" % (self.volume.user, self.path, self.file.oshash)
@property @property
def public_id(self): def public_id(self):
@ -618,8 +619,8 @@ class Frame(models.Model):
file = models.ForeignKey(File, related_name="frames") file = models.ForeignKey(File, related_name="frames")
position = models.FloatField() position = models.FloatField()
frame = models.ImageField(default=None, null=True, upload_to=frame_path) frame = models.ImageField(default=None, null=True, upload_to=frame_path)
width = models.IntegerField(default = 0) width = models.IntegerField(default=0)
height = models.IntegerField(default = 0) height = models.IntegerField(default=0)
def save(self, *args, **kwargs): def save(self, *args, **kwargs):
if self.frame: if self.frame:
@ -636,8 +637,8 @@ def delete_frame(sender, **kwargs):
f.frame.delete(save=False) f.frame.delete(save=False)
pre_delete.connect(delete_frame, sender=Frame) pre_delete.connect(delete_frame, sender=Frame)
def stream_path(f, x):
def stream_path(f, x): return f.path(x) return f.path(x)
class Stream(models.Model): class Stream(models.Model):
@ -696,7 +697,8 @@ class Stream(models.Model):
if resolution <= self.resolution: if resolution <= self.resolution:
for f in config['formats']: for f in config['formats']:
derivative, created = Stream.objects.get_or_create(file=self.file, derivative, created = Stream.objects.get_or_create(file=self.file,
resolution=resolution, format=f) resolution=resolution,
format=f)
if created: if created:
derivative.source = self derivative.source = self
derivative.save() derivative.save()

22
pandora/archive/tasks.py
View file

@ -24,31 +24,31 @@ def get_or_create_file(volume, f, user, item=None):
if item: if item:
file.item = item file.item = item
else: else:
file.item = None #gets pupulated later via update_info file.item = None # gets pupulated later via update_info
file.save() file.save()
return file return file
def update_or_create_instance(volume, f): def update_or_create_instance(volume, f):
#instance with oshash exists # instance with oshash exists
instance = models.Instance.objects.filter(file__oshash=f['oshash'], volume=volume) instance = models.Instance.objects.filter(file__oshash=f['oshash'], volume=volume)
if instance.count()>0: if instance.count():
instance = instance[0] instance = instance[0]
updated = False updated = False
for key in _INSTANCE_KEYS: for key in _INSTANCE_KEYS:
if f[key] != getattr(instance, key): if f[key] != getattr(instance, key):
setattr(instance, key, f[key]) setattr(instance, key, f[key])
updated=True updated = True
if updated: if updated:
instance.ignore = False instance.ignore = False
instance.save() instance.save()
instance.file.save() instance.file.save()
else: else:
instance = models.Instance.objects.filter(path=f['path'], volume=volume) instance = models.Instance.objects.filter(path=f['path'], volume=volume)
if instance.count()>0: if instance.count():
#same path, other oshash, keep path/item mapping, remove instance # same path, other oshash, keep path/item mapping, remove instance
item = instance[0].file.item item = instance[0].file.item
instance.delete() instance.delete()
else: #new instance else: # new instance
item = None item = None
instance = models.Instance() instance = models.Instance()
@ -144,14 +144,14 @@ def extract_stream(fileId):
def extract_derivatives(fileId, rebuild=False): def extract_derivatives(fileId, rebuild=False):
file = models.File.objects.get(id=fileId) file = models.File.objects.get(id=fileId)
streams = file.streams.filter(source=None) streams = file.streams.filter(source=None)
if streams.count() > 0: if streams.count():
streams[0].extract_derivatives(rebuild) streams[0].extract_derivatives(rebuild)
return True return True
@task(queue="encoding") @task(queue="encoding")
def update_stream(id): def update_stream(id):
s = models.Stream.objects.get(pk=id) s = models.Stream.objects.get(pk=id)
if not glob("%s*"%s.timeline_prefix): if not glob("%s*" % s.timeline_prefix):
s.make_timeline() s.make_timeline()
if not s.color: if not s.color:
s.cuts = tuple(extract.cuts(s.timeline_prefix)) s.cuts = tuple(extract.cuts(s.timeline_prefix))
@ -161,11 +161,11 @@ def update_stream(id):
s.file.selected = True s.file.selected = True
s.file.save() s.file.save()
s.file.item.update_timeline() s.file.item.update_timeline()
#make sure all derivatives exist # make sure all derivatives exist
s.extract_derivatives() s.extract_derivatives()
s.file.item.save() s.file.item.save()
#update clips # update clips
for c in s.file.item.clips.all(): for c in s.file.item.clips.all():
c.update_calculated_values() c.update_calculated_values()
c.save() c.save()

40
pandora/archive/views.py
View file

@ -94,7 +94,7 @@ def update(request, data):
files = all_files.filter(file__available=False) files = all_files.filter(file__available=False)
if volume: if volume:
files = files.filter(volume=volume) files = files.filter(volume=volume)
response['data']['info'] = [f.file.oshash for f in all_files.filter(Q(file__info='{}')|Q(file__size=0))] response['data']['info'] = [f.file.oshash for f in all_files.filter(Q(file__info='{}') | Q(file__size=0))]
response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True, response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True,
file__available=False, file__available=False,
file__wanted=True)] file__wanted=True)]
@ -135,13 +135,13 @@ def upload(request, data=None):
f.frames.all().delete() f.frames.all().delete()
for frame in request.FILES.getlist('frame'): for frame in request.FILES.getlist('frame'):
name = frame.name name = frame.name
#float required? # float required?
position = float(os.path.splitext(name)[0]) position = float(os.path.splitext(name)[0])
fr, created = models.Frame.objects.get_or_create(file=f, position=position) fr, created = models.Frame.objects.get_or_create(file=f, position=position)
if fr.frame: if fr.frame:
fr.frame.delete() fr.frame.delete()
fr.frame.save(name, frame) fr.frame.save(name, frame)
os.chmod(fr.frame.path, 0644) os.chmod(fr.frame.path, 0o644)
fr.save() fr.save()
f.item.select_frame() f.item.select_frame()
f.item.save() f.item.save()
@ -152,7 +152,7 @@ def upload(request, data=None):
f.data.delete() f.data.delete()
f.data.save('data.raw', request.FILES['file']) f.data.save('data.raw', request.FILES['file'])
f.save() f.save()
os.chmod(f.data.path, 0644) os.chmod(f.data.path, 0o644)
item.tasks.load_subtitles.delay(f.item.public_id) item.tasks.load_subtitles.delay(f.item.public_id)
response = json_response(text='file saved') response = json_response(text='file saved')
else: else:
@ -235,20 +235,20 @@ def firefogg_upload(request):
resolution, format = profile.split('p.') resolution, format = profile.split('p.')
resolution = int(resolution) resolution = int(resolution)
if resolution not in config['resolutions'] \ if resolution not in config['resolutions'] \
or format not in config['formats']: or format not in config['formats']:
response = json_response(status=500, text='invalid profile') response = json_response(status=500, text='invalid profile')
return render_to_json_response(response) return render_to_json_response(response)
#handle video upload # handle video upload
if request.method == 'POST': if request.method == 'POST':
#post next chunk # post next chunk
if 'chunk' in request.FILES and oshash: if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash) f = get_object_or_404(models.File, oshash=oshash)
if f.editable(request.user): if f.editable(request.user):
def save_chunk(chunk, offset, done): def save_chunk(chunk, offset, done):
return f.save_chunk_stream(chunk, offset, resolution, format, done) return f.save_chunk_stream(chunk, offset, resolution, format, done)
response = process_chunk(request, save_chunk) response = process_chunk(request, save_chunk)
response['resultUrl'] = request.build_absolute_uri('/%s'%f.item.public_id) response['resultUrl'] = request.build_absolute_uri('/%s' % f.item.public_id)
if response.get('done'): if response.get('done'):
f.uploading = False f.uploading = False
if response['result'] == 1: if response['result'] == 1:
@ -258,16 +258,16 @@ def firefogg_upload(request):
f.queued = False f.queued = False
f.wanted = True f.wanted = True
f.save() f.save()
#FIXME: this fails badly if rabbitmq goes down # FIXME: this fails badly if rabbitmq goes down
try: try:
t = f.process_stream() t = f.process_stream()
response['resultUrl'] = t.task_id response['resultUrl'] = t.task_id
except: except:
pass pass
return render_to_json_response(response) return render_to_json_response(response)
#init upload # init upload
elif oshash: elif oshash:
#404 if oshash is not know, files must be registered via update api first # 404 if oshash is not know, files must be registered via update api first
f = get_object_or_404(models.File, oshash=oshash) f = get_object_or_404(models.File, oshash=oshash)
if f.editable(request.user): if f.editable(request.user):
f.streams.all().delete() f.streams.all().delete()
@ -309,18 +309,18 @@ def direct_upload(request):
file.queued = False file.queued = False
file.wanted = True file.wanted = True
file.save() file.save()
#try/execpt so it does not fail if rabitmq is down # try/execpt so it does not fail if rabitmq is down
try: try:
t = file.extract_stream() t = file.extract_stream()
response['resultUrl'] = t.task_id response['resultUrl'] = t.task_id
except: except:
pass pass
return render_to_json_response(response) return render_to_json_response(response)
#init upload # init upload
else: else:
file, created = models.File.objects.get_or_create(oshash=oshash) file, created = models.File.objects.get_or_create(oshash=oshash)
if file.editable(request.user): if file.editable(request.user):
#remove previous uploads # remove previous uploads
if not created: if not created:
file.streams.all().delete() file.streams.all().delete()
file.delete_frames() file.delete_frames()
@ -353,7 +353,7 @@ def getTaskStatus(request, data):
} }
notes: To be deprecated, will be wrapped in regular API call. notes: To be deprecated, will be wrapped in regular API call.
''' '''
#FIXME: should check if user has permissions to get status # FIXME: should check if user has permissions to get status
if 'id' in data: if 'id' in data:
task_id = data['id'] task_id = data['id']
elif 'taskId' in data: elif 'taskId' in data:
@ -477,7 +477,7 @@ def editMedia(request, data):
models.Instance.objects.filter(file__oshash__in=dont_ignore).update(ignore=False) models.Instance.objects.filter(file__oshash__in=dont_ignore).update(ignore=False)
if ignore or dont_ignore: if ignore or dont_ignore:
files = models.File.objects.filter(oshash__in=ignore+dont_ignore) files = models.File.objects.filter(oshash__in=ignore+dont_ignore)
#FIXME: is this to slow to run sync? # FIXME: is this to slow to run sync?
for i in Item.objects.filter(files__in=files).distinct(): for i in Item.objects.filter(files__in=files).distinct():
i.update_selected() i.update_selected()
i.update_wanted() i.update_wanted()
@ -624,7 +624,7 @@ def findMedia(request, data):
qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by) qs = qs.values('value').annotate(items=Count('id')).order_by(*order_by)
if 'positions' in query: if 'positions' in query:
#FIXME: this does not scale for larger results # FIXME: this does not scale for larger results
response['data']['positions'] = {} response['data']['positions'] = {}
ids = [j['value'] for j in qs] ids = [j['value'] for j in qs]
response['data']['positions'] = utils.get_positions(ids, query['positions']) response['data']['positions'] = utils.get_positions(ids, query['positions'])
@ -635,7 +635,7 @@ def findMedia(request, data):
else: else:
response['data']['items'] = qs.count() response['data']['items'] = qs.count()
elif 'positions' in query: elif 'positions' in query:
#FIXME: this does not scale for larger results # FIXME: this does not scale for larger results
qs = models.File.objects.filter(item__in=query['qs']) qs = models.File.objects.filter(item__in=query['qs'])
qs = _order_query(qs, query['sort']) qs = _order_query(qs, query['sort'])
@ -651,7 +651,7 @@ def findMedia(request, data):
keys = query['keys'] keys = query['keys']
qs = qs[query['range'][0]:query['range'][1]] qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [f.json(keys) for f in qs] response['data']['items'] = [f.json(keys) for f in qs]
else: # otherwise stats else: # otherwise stats
items = query['qs'] items = query['qs']
files = models.File.objects.filter(item__in=query['qs']) files = models.File.objects.filter(item__in=query['qs'])
response['data']['items'] = files.count() response['data']['items'] = files.count()
@ -659,7 +659,7 @@ def findMedia(request, data):
actions.register(findMedia) actions.register(findMedia)
def parsePath(request, data): #parse path and return info def parsePath(request, data): # parse path and return info
''' '''
Parses a path Parses a path
takes { takes {