0x2620/pandora
2
1
Fork 1
Code Issues 1.3k Pull requests Releases Wiki Activity

rewrite file/instance backend

Browse source
This commit is contained in:
j 2013-02-08 17:01:26 +00:00
parent 04a391a442
commit 7951c42609
3 changed files with 72 additions and 75 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
pandora/archive/models.py
View file

@ -18,18 +18,31 @@ import ox
import chardet import chardet
from item import utils from item import utils
import item.models
from person.models import get_name_sort from person.models import get_name_sort
import extract import extract
class File(models.Model): class File(models.Model):
AV_INFO = (
'duration', 'video', 'audio', 'oshash', 'size',
)
PATH_INFO = (
'episodes', 'extension', 'language', 'part', 'partTitle', 'version'
)
ITEM_INFO = (
'title', 'director', 'year',
'season', 'episode', 'episodeTitle',
'seriesTitle', 'seriesYear'
)
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True) modified = models.DateTimeField(auto_now=True)
oshash = models.CharField(max_length=16, unique=True) oshash = models.CharField(max_length=16, unique=True)
item = models.ForeignKey("item.Item", related_name='files') item = models.ForeignKey("item.Item", related_name='files', null=True)
path = models.CharField(max_length=2048, default="") # canoncial path/file path = models.CharField(max_length=2048, default="") # canoncial path/file
sort_path = models.CharField(max_length=2048, default="") # sort name sort_path = models.CharField(max_length=2048, default="") # sort name
@ -127,28 +140,15 @@ class File(models.Model):
if self.framerate: if self.framerate:
self.pixels = int(self.width * self.height * float(utils.parse_decimal(self.framerate)) * self.duration) self.pixels = int(self.width * self.height * float(utils.parse_decimal(self.framerate)) * self.duration)
def parse_instance_path(self):
if self.instances.count():
path = self.instances.all()[0].path
data = ox.movie.parse_path(path)
for key in (
'normalizedPath', 'isEpisode',
'title', 'director', 'directorSort', 'year',
'season', 'episode', 'episodeTitle',
'seriesTitle', 'seriesYear'
):
del data[key]
self.path_info = data
def get_path_info(self): def get_path_info(self):
data = self.path_info.copy() data = {}
for key in ( for key in self.PATH_INFO:
'title', 'director', 'year', data[key] = self.info.get(key, None)
'season', 'episode', 'episodeTitle', if self.item:
'seriesTitle', 'seriesYear' for key in self.ITEM_INFO:
): data[key] = self.item.get(key)
data[key] = self.item.get(key) data['directorSort'] = [get_name_sort(n) for n in self.item.get('director', [])]
data['directorSort'] = [get_name_sort(n) for n in self.item.get('director', [])]
data['isEpisode'] = data.get('season') != None \ data['isEpisode'] = data.get('season') != None \
or data.get('episode') != None \ or data.get('episode') != None \
or data.get('episodes') != [] or data.get('episodes') != []
@ -160,12 +160,10 @@ class File(models.Model):
for type in ox.movie.EXTENSIONS: for type in ox.movie.EXTENSIONS:
if data['extension'] in ox.movie.EXTENSIONS[type]: if data['extension'] in ox.movie.EXTENSIONS[type]:
data['type'] = type data['type'] = type
if 'part' in data and not data['part']:
del data['part']
return data return data
def normalize_path(self): def normalize_path(self):
#FIXME: always use path_info #FIXME: always use format_path
if settings.USE_IMDB: if settings.USE_IMDB:
return ox.movie.format_path(self.get_path_info()) return ox.movie.format_path(self.get_path_info())
else: else:
@ -174,24 +172,31 @@ class File(models.Model):
path = self.instances.all()[0].path path = self.instances.all()[0].path
return path return path
def save(self, *args, **kwargs): def update_info(self, info, user):
if self.id and not self.path_info and self.instances.count(): #populate name sort with director if unknown
self.parse_info() if info.get('director') and info.get('directorSort'):
self.parse_instance_path() for name, sortname in zip(info['director'], info['directorSort']):
self.path = self.normalize_path() get_name_sort(name, sortname)
self.item = item.models.get_item(info, user)
for key in self.AV_INFO + self.PATH_INFO:
if key in info:
self.info[key] = info[key]
self.parse_info()
data = self.get_path_info() def save(self, *args, **kwargs):
self.extension = data.get('extension') if self.id and self.info:
self.language = data.get('language') self.path = self.normalize_path()
self.part = ox.sort_string(unicode(data.get('part', ''))) if self.item:
self.part_title = ox.sort_string(unicode(data.get('partTitle')) or '') data = self.get_path_info()
self.type = data.get('type') or 'unknown' self.extension = data.get('extension')
self.version = data.get('version') self.language = data.get('language')
self.part = ox.sort_string(unicode(data.get('part') or ''))
self.part_title = ox.sort_string(unicode(data.get('partTitle')) or '')
self.type = data.get('type') or 'unknown'
self.version = data.get('version')
if self.path: if self.path:
self.path = self.normalize_path()
self.sort_path = utils.sort_string(self.path) self.sort_path = utils.sort_string(self.path)
self.is_audio = self.type == 'audio' self.is_audio = self.type == 'audio'
self.is_video = self.type == 'video' self.is_video = self.type == 'video'
self.is_subtitle = self.path.endswith('.srt') self.is_subtitle = self.path.endswith('.srt')
@ -238,7 +243,7 @@ class File(models.Model):
p = user.get_profile() p = user.get_profile()
return p.get_level() in ('admin', 'staff') or \ return p.get_level() in ('admin', 'staff') or \
self.instances.filter(volume__user=user).count() > 0 or \ self.instances.filter(volume__user=user).count() > 0 or \
self.item.user == user (not self.item or self.item.user == user)
def save_chunk(self, chunk, chunk_id=-1, done=False): def save_chunk(self, chunk, chunk_id=-1, done=False):
if not self.available: if not self.available:
@ -286,8 +291,8 @@ class File(models.Model):
'videoCodec': self.video_codec, 'videoCodec': self.video_codec,
'wanted': self.wanted, 'wanted': self.wanted,
} }
for key in ('part', 'partTitle', 'version', 'language', 'extension'): for key in self.PATH_INFO:
data[key] = self.path_info.get(key) data[key] = self.info.get(key)
data['users'] = list(set([i['user'] for i in data['instances']])) data['users'] = list(set([i['user'] for i in data['instances']]))
if keys: if keys:
for k in data.keys(): for k in data.keys():

15
pandora/archive/tasks.py
View file

@ -7,20 +7,13 @@ import ox
from django.conf import settings from django.conf import settings
from item.models import get_item, Item from item.models import Item
import item.tasks import item.tasks
from person.models import get_name_sort
import models import models
import extract import extract
_INSTANCE_KEYS = ('mtime', 'path') _INSTANCE_KEYS = ('mtime', 'path')
def get_or_create_item(volume, info, user):
item_info = ox.parse_movie_path(info['path'])
if item_info.get('director') and item_info.get('directorSort'):
for name, sortname in zip(item_info['director'], item_info['directorSort']):
get_name_sort(name, sortname)
return get_item(item_info, user)
def get_or_create_file(volume, f, user, item=None): def get_or_create_file(volume, f, user, item=None):
try: try:
@ -29,10 +22,11 @@ def get_or_create_file(volume, f, user, item=None):
file = models.File() file = models.File()
file.oshash = f['oshash'] file.oshash = f['oshash']
file.path = f['path'] file.path = f['path']
file.path = f['path']
if item: if item:
file.item = item file.item = item
else: else:
file.item = get_or_create_item(volume, f, user) file.item = None #gets pupulated later via update_info
file.save() file.save()
return file return file
@ -66,7 +60,8 @@ def update_or_create_instance(volume, f):
setattr(instance, key, f[key]) setattr(instance, key, f[key])
instance.save() instance.save()
instance.file.save() instance.file.save()
instance.file.item.update_wanted() if instance.file.item:
instance.file.item.update_wanted()
return instance return instance
@task(ignore_results=True, queue='default') @task(ignore_results=True, queue='default')

45
pandora/archive/views.py
View file

@ -7,7 +7,7 @@ from datetime import datetime
from django import forms from django import forms
from django.shortcuts import get_object_or_404, redirect from django.shortcuts import get_object_or_404, redirect
from django.conf import settings from django.conf import settings
from django.db.models import Count from django.db.models import Count, Q
import ox import ox
from ox.utils import json from ox.utils import json
@ -43,9 +43,12 @@ actions.register(removeVolume, cache=False)
@login_required_json @login_required_json
def update(request): def update(request):
''' '''
2 calls possible: 2 steps:
volume/files send files
info {volume: 'Videos', files: [{oshash:, path:, mtime:, ,,}]}
send info about changed/new files
{volume: 'Videos', info: {oshash: {...}]}
call volume/files first and fill in requested info after that call volume/files first and fill in requested info after that
param data { param data {
@ -56,12 +59,13 @@ def update(request):
], ],
info: {oshash: object} info: {oshash: object}
} }
return { return {
status: {'code': int, 'text': string}, status: {'code': int, 'text': string},
data: { data: {
info: list, info: list, // list of files that need info
data: list, data: list, // list of flies that should be encoded to highest profile and uploaded
file: list file: list // list of files that should be uploaded as is
} }
} }
''' '''
@ -82,23 +86,16 @@ def update(request):
user_profile.save() user_profile.save()
if 'info' in data: if 'info' in data:
for oshash in data['info']: for f in models.File.objects.filter(oshash__in=data['info'].keys()):
info = data['info'][oshash] if not f.info:
f = models.File.objects.filter(oshash=oshash) f.update_info(data['info'][f.oshash], user)
if f.count()>0: f.save()
f = f[0]
if not f.info:
for key in ('atime', 'mtime', 'ctime'):
if key in info:
del info[key]
f.info = info
f.parse_info()
f.save()
if not upload_only: if not upload_only:
all_files = models.Instance.objects.filter(volume__user=user) all_files = models.Instance.objects.filter(volume__user=user)
files = all_files.filter(file__available=False) files = all_files.filter(file__available=False)
if volume: if volume:
files = files.filter(volume=volume) files = files.filter(volume=volume)
#fixme: might be better to check for file__path_info
response['data']['info'] = [f.file.oshash for f in all_files.filter(file__info='{}')] response['data']['info'] = [f.file.oshash for f in all_files.filter(file__info='{}')]
response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True, response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True,
file__available=False, file__available=False,
@ -211,11 +208,10 @@ def addFile(request):
extension = extension[-1] extension = extension[-1]
else: else:
extension = 'webm' extension = 'webm'
f.path_info = {
'extension': extension
}
f.selected = True f.selected = True
f.info = data['info'] f.info = data['info']
f.info['extension'] = extension
self.parse_info()
f.save() f.save()
response['data']['item'] = i.itemId response['data']['item'] = i.itemId
response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.itemId) response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.itemId)
@ -408,8 +404,9 @@ def editFile(request):
f.instances.update(ignore=data['ignore']) f.instances.update(ignore=data['ignore'])
f.save() f.save()
#FIXME: is this to slow to run sync? #FIXME: is this to slow to run sync?
f.item.update_selected() if f.item:
f.item.update_wanted() f.item.update_selected()
f.item.update_wanted()
for key in ('episodes', 'extension', 'language', 'part', 'partTitle', 'version'): for key in ('episodes', 'extension', 'language', 'part', 'partTitle', 'version'):
if key in data: if key in data:
f.path_info[key] = data[key] f.path_info[key] = data[key]