cablegates/pandora/item/models.py

943 lines
34 KiB
Python
Raw Normal View History

2009-06-08 16:08:59 +00:00
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
2010-08-07 14:31:20 +00:00
from __future__ import division, with_statement
2010-02-03 12:05:38 +00:00
from datetime import datetime
2009-06-08 16:08:59 +00:00
import os.path
2010-09-03 13:28:44 +00:00
import math
import random
2010-02-03 12:05:38 +00:00
import re
2010-09-03 13:28:44 +00:00
import subprocess
import unicodedata
from glob import glob
2009-06-08 16:08:59 +00:00
from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User
2009-12-31 15:04:32 +00:00
from django.core.files.base import ContentFile
from django.utils import simplejson as json
from django.conf import settings
from ox.django import fields
2010-07-07 22:46:41 +00:00
import ox
from ox import stripTags
from ox.normalize import canonicalTitle, canonicalName
2010-02-06 08:28:35 +00:00
from firefogg import Firefogg
2009-06-08 16:08:59 +00:00
2009-08-01 14:14:54 +00:00
import managers
2010-01-16 20:42:11 +00:00
import load
2010-02-03 12:05:38 +00:00
import utils
import tasks
2010-09-03 13:28:44 +00:00
from archive import extract
2009-06-08 16:08:59 +00:00
2010-11-28 16:03:23 +00:00
from annotaion.models import Annotation, Layer
from person.models import get_name_sort, Person
2010-01-16 20:42:11 +00:00
def siteJson():
r = {}
r['findKeys'] = [{"id": "all", "title": "All"}]
for p in Property.objects.all():
if p.find:
title = p.title
if not title:
title = p.name.capitalize()
f = {"id": p.name, "title": title}
f['autocomplete'] = p.autocomplete
r['findKeys'].append(f)
r['groups'] = [p.name for p in Property.objects.filter(group=True)]
2010-11-28 16:31:53 +00:00
r['layers'] = [l.json() for l in Layer.objects.all()]
2010-11-28 16:03:23 +00:00
r['itemViews'] = [
{"id": "info", "title": "Info"},
{"id": "statistics", "title": "Statistics"},
{"id": "clips", "title": "Clips"},
{"id": "timeline", "title": "Timeline"},
{"id": "map", "title": "Map"},
{"id": "calendar", "title": "Calendar"},
{"id": "files", "title": "Files", "admin": True}
]
r['listViews'] = [
{"id": "list", "title": "as List"},
{"id": "icons", "title": "as Icons"},
{"id": "info", "title": "with Info"},
{"id": "clips", "title": "with Clips"},
{"id": "timelines", "title": "with Timelines"},
{"id": "maps", "title": "with Maps"},
{"id": "calendars", "title": "with Calendars"},
{"id": "clip", "title": "as Clips"},
{"id": "map", "title": "on Map"},
{"id": "calendar", "title": "on Calendar"}
]
r['site'] = {
"name": settings.SITENAME,
"id": settings.SITEID,
"url": settings.URL
}
r['sections'] = [
{"id": "history", "title": "History"},
{"id": "lists", "title": "My Lists"},
{"id": "public", "title": "Public Lists"},
{"id": "featured", "title": "Featured Lists"}
]
r['sortKeys'] = []
for p in Property.objects.exclude(sort=''):
title = p.title
if not title:
title = p.name.capitalize()
f = {
"id": p.name,
"title": title,
"operator": p.operator,
"align": p.align,
"width": p.width,
}
if not p.removable:
f['removable'] = False
r['sortKeys'].append(f)
r['sortKeys'].append([{"id": "id", "title": "ID", "operator": "", "align": "left", "width": 90}])
r['totals'] = [{"id": "items"}]
for p in Property.objects.filter(totals=True):
f = {'id': p.name, 'admin': p.admin}
r['totals'].append(f)
#FIXME: defaults should also be populated from properties
r["user"] = {
"group": "guest",
"preferences": {},
"ui": {
"columns": ["id"] + [p.name for p in Property.objects.filter(default=True)],
"findQuery": {"conditions": [], "operator": ""},
"groupsQuery": {"conditions": [], "operator": "|"},
"groupsSize": 128,
2010-12-06 22:32:17 +00:00
"itemView": "timeline",
"listQuery": {"conditions": [], "operator": ""},
"listsSize": 192,
"listView": "icons",
"sections": ["history", "lists", "public", "featured"],
"showGroups": True,
"showInfo": True,
"showLists": True,
"showMovies": True,
"sort": settings.DEFAULT_SORT,
"theme": settings.DEFAULT_THEME
},
"username": ""
}
return r
def get_item(info):
2010-01-16 20:42:11 +00:00
'''
info dict with:
imdbId, title, director, episode_title, season, series
'''
2010-01-22 23:57:06 +00:00
if 'imdbId' in info and info['imdbId']:
2010-01-16 20:42:11 +00:00
try:
2010-09-23 16:01:48 +00:00
item = Item.objects.get(itemId=info['imdbId'])
except Item.DoesNotExist:
item = Item(itemId=info['imdbId'])
if 'title' in info and 'directors' in info:
item.external_data = {
'title': info['title'],
'directors': info['directors'],
'year': info.get('year', '')
}
2010-08-07 14:31:20 +00:00
#FIXME: this should be done async
2010-09-23 16:01:48 +00:00
#item.save()
#tasks.update_imdb.delay(item.itemId)
item.update_imdb()
tasks.update_poster.delay(item.itemId)
2010-01-16 20:42:11 +00:00
else:
2010-11-26 17:11:26 +00:00
q = Item.objects.filter(find__key='title', find__value=info['title'])
2010-01-16 20:42:11 +00:00
if q.count() > 1:
print "FIXME: check more than title here!!?"
2010-09-23 16:01:48 +00:00
item = q[0]
2010-01-16 20:42:11 +00:00
else:
try:
2010-09-23 16:01:48 +00:00
item = Item.objects.get(itemId=info['oxdbId'])
except Item.DoesNotExist:
item = Item()
item.data = {
'title': info['title'],
'directors': info['directors'],
'year': info.get('year', '')
}
2010-09-23 16:01:48 +00:00
item.itemId = info['oxdbId']
for key in ('episode_title', 'series_title', 'season', 'episode'):
2010-08-07 14:31:20 +00:00
if key in info and info[key]:
item.data[key] = info[key]
try:
existing_item = Item.objects.get(oxdbId=item.oxdb_id())
item = existing_item
except Item.DoesNotExist:
item.save()
2010-09-23 16:01:48 +00:00
return item
2010-01-16 20:42:11 +00:00
2010-11-25 15:21:23 +00:00
class Property(models.Model):
class Meta:
ordering = ('position', )
verbose_name_plural = "Properties"
name = models.CharField(null=True, max_length=255, unique=True)
title = models.CharField(null=True, max_length=255, blank=True)
#text, string, string from list(fixme), event, place, person
type = models.CharField(null=True, max_length=255)
array = models.BooleanField(default=False)
position = models.IntegerField(default=0)
width = models.IntegerField(default=180)
align = models.CharField(null=True, max_length=255, default='left')
operator = models.CharField(null=True, max_length=5, default='', blank=True)
default = models.BooleanField('Enabled by default', default=False)
removable = models.BooleanField(default=True)
#sort values: title, string, integer, float, date
sort = models.CharField(null=True, max_length=255, blank=True)
find = models.BooleanField(default=False)
autocomplete = models.BooleanField(default=False)
group = models.BooleanField(default=False)
totals = models.BooleanField(default=False)
admin = models.BooleanField(default=False)
2010-11-25 15:21:23 +00:00
def __unicode__(self):
if self.title:
return self.title
return self.name
def json(self):
j = {}
for key in ('type', 'sort', 'title', 'array', 'totals', 'admin'):
value = getattr(self, key)
if value:
j[key] = value
return j
def save(self, *args, **kwargs):
if not self.title:
self.title = self.name.capitalize()
super(Property, self).save(*args, **kwargs)
2010-09-23 16:01:48 +00:00
class Item(models.Model):
2010-07-16 10:13:16 +00:00
person_keys = ('director', 'writer', 'producer', 'editor', 'cinematographer', 'actor', 'character')
2010-07-15 15:55:10 +00:00
facet_keys = person_keys + ('country', 'language', 'genre', 'keyword')
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
2010-01-16 20:42:11 +00:00
published = models.DateTimeField(default=datetime.now, editable=False)
2009-06-08 16:08:59 +00:00
#only items that have data from files are available,
#this is indicated by setting available to True
2009-12-31 15:04:32 +00:00
available = models.BooleanField(default=False, db_index=True)
2010-09-23 16:01:48 +00:00
itemId = models.CharField(max_length=128, unique=True, blank=True)
oxdbId = models.CharField(max_length=42, unique=True, blank=True)
2010-11-27 02:33:31 +00:00
external_data = fields.DictField(default={}, editable=False)
data = fields.DictField(default={}, editable=False)
json = fields.DictField(default={}, editable=False)
2010-12-07 23:24:35 +00:00
poster = models.ImageField(default=None, blank=True, upload_to=lambda i, x: i.path("poster.jpg"))
2010-11-27 02:33:31 +00:00
poster_url = models.TextField(blank=True)
poster_height = models.IntegerField(default=0)
poster_width = models.IntegerField(default=0)
poster_frame = models.FloatField(default=-1)
#stream related fields
stream_aspect = models.FloatField(default=4/3)
2010-09-23 16:01:48 +00:00
objects = managers.ItemManager()
2009-08-01 14:14:54 +00:00
def get(self, key, default=None):
if self.data and key in self.data:
return self.data[key]
if self.external_data and key in self.external_data:
return self.external_data[key]
return default
def editable(self, user):
#FIXME: make permissions work
return False
def edit(self, data):
#FIXME: how to map the keys to the right place to write them to?
for key in data:
if key != 'id':
setattr(self.data, key, data[key])
self.oxdb.save()
self.save()
def reviews(self):
reviews = self.get('reviews', [])
_reviews = {}
for r in reviews:
for url in settings.REVIEW_WHITELIST:
if url in r[0]:
_reviews[settings.REVIEW_WHITELIST[url]] = r[0]
return _reviews
def update_imdb(self):
2010-09-23 16:01:48 +00:00
if len(self.itemId) == 7:
self.external_data = ox.web.imdb.Imdb(self.itemId)
self.save()
2009-06-08 16:08:59 +00:00
def __unicode__(self):
year = self.get('year')
if year:
return u'%s (%s)' % (self.get('title'), self.get('year'))
return self.get('title')
2010-09-14 14:10:37 +00:00
def get_absolute_url(self):
return '/%s' % self.itemId
2010-09-14 14:10:37 +00:00
2009-06-08 16:08:59 +00:00
def save(self, *args, **kwargs):
self.json = self.get_json()
self.oxdbId = self.oxdb_id()
2010-07-05 12:07:59 +00:00
2010-09-06 20:31:12 +00:00
if self.poster:
self.poster_height = self.poster.height
self.poster_width = self.poster.width
else:
self.poster_height = 128
self.poster_width = 80
2010-09-23 16:01:48 +00:00
super(Item, self).save(*args, **kwargs)
self.update_find()
self.update_sort()
self.update_facets()
2009-06-08 16:08:59 +00:00
2010-09-12 14:23:23 +00:00
def delete(self, *args, **kwargs):
self.delete_poster()
for f in glob("%s*"%self.timeline_prefix):
os.unlink(f)
for f in glob("%sstrip*"%self.timeline_prefix[:-8]):
os.unlink(f)
2010-09-23 16:01:48 +00:00
super(Item, self).delete(*args, **kwargs)
2010-09-12 14:23:23 +00:00
def merge_with(self, other):
2010-09-12 14:23:23 +00:00
'''
move all related tables to other and delete self
'''
for stream in self.streams.all():
2010-09-23 16:01:48 +00:00
stream.item = other
2010-09-12 14:23:23 +00:00
stream.save()
for l in self.lists.all():
2010-09-23 16:01:48 +00:00
l.items.remove(self)
if l.items.filter(id=other.id) == 0:
l.items.add(other)
2010-09-12 14:23:23 +00:00
#FIXME: should this really happen for layers?
for l in self.layer.all():
2010-09-23 16:01:48 +00:00
l.items.remove(self)
if l.items.filter(id=other.id) == 0:
l.items.add(other)
2010-09-12 14:23:23 +00:00
if hasattr(self, 'files'):
for f in self.files.all():
2010-09-23 16:01:48 +00:00
f.item = other
2010-09-12 14:23:23 +00:00
f.save()
self.delete()
other.save()
2010-09-10 15:12:22 +00:00
'''
JSON cache related functions
'''
#FIXME: this should not be used
2009-06-08 16:08:59 +00:00
_public_fields = {
2010-09-23 16:01:48 +00:00
'itemId': 'id',
2009-06-08 16:08:59 +00:00
'title': 'title',
'year': 'year',
'runtime': 'runtime',
'release_date': 'release_date',
'countries': 'country',
'directors': 'director',
2010-09-06 21:19:59 +00:00
'writers': 'writer',
'editors': 'editor',
'producers': 'producer',
'cinematographer': 'cinematographer',
'languages': 'language',
'genres': 'genre',
'keywords': 'keyword',
2009-06-08 16:08:59 +00:00
'cast': 'cast',
'series_title': 'series_title',
'episode_title': 'episode_title',
'season': 'season',
'episode': 'episode',
'reviews': 'reviews',
2009-08-01 14:14:54 +00:00
'trivia': 'trivia',
'rating': 'rating',
'votes': 'votes',
2009-08-16 12:23:29 +00:00
'alternative_titles': 'alternative_titles',
2010-07-05 08:30:17 +00:00
'connections_json': 'connections',
2009-06-08 16:08:59 +00:00
}
2010-11-27 02:33:31 +00:00
2010-09-07 14:05:38 +00:00
def get_poster(self):
2010-09-06 20:45:11 +00:00
poster = {}
poster['width'] = self.poster_width
poster['height'] = self.poster_height
2010-09-23 16:01:48 +00:00
poster['url'] = '/%s/poster.jpg' % self.itemId
2010-09-07 14:05:38 +00:00
'''
2010-09-06 20:45:11 +00:00
if self.poster:
poster['url'] = self.poster.url
else:
poster['url'] = self.poster_url
2010-09-07 14:05:38 +00:00
'''
2010-09-06 20:45:11 +00:00
return poster
2010-09-10 15:12:22 +00:00
def get_posters(self):
posters = {}
for p in self.poster_urls.all():
if p.service not in posters:
posters[p.service] = []
posters[p.service].append({'url': p.url, 'width': p.width, 'height': p.height})
local_posters = self.local_posters().keys()
if local_posters:
posters['local'] = []
for p in local_posters:
url = p.replace(settings.MEDIA_ROOT, settings.MEDIA_URL)
width = 640
height = 1024
posters['local'].append({'url': url, 'width': width, 'height': height})
return posters
def get_stream(self):
stream = {}
if self.streams.all().count():
s = self.streams.all()[0]
if s.video and s.info:
stream['duration'] = s.info['duration']
if 'video' in s.info and s.info['video']:
stream['aspectRatio'] = s.info['video'][0]['width'] / s.info['video'][0]['height']
2010-09-14 15:03:10 +00:00
if settings.XSENDFILE or settings.XACCELREDIRECT:
2010-09-23 16:01:48 +00:00
stream['baseUrl'] = '/%s' % self.itemId
2010-09-14 15:03:10 +00:00
else:
stream['baseUrl'] = os.path.dirname(s.video.url)
2010-09-10 15:12:22 +00:00
stream['profiles'] = list(set(map(lambda s: int(os.path.splitext(s['profile'])[0][:-1]), self.streams.all().values('profile'))))
return stream
2010-09-18 14:44:35 +00:00
def get_layers(self):
layers = {}
layers['cuts'] = self.data.get('cuts', {})
2010-09-18 14:44:35 +00:00
layers['subtitles'] = {}
2010-11-28 16:03:23 +00:00
#FIXME: subtitles should be stored in Annotation
2010-09-18 14:44:35 +00:00
qs = self.files.filter(is_subtitle=True, is_main=True, available=True)
if qs.count()>0:
layers['subtitles'] = qs[0].srt()
return layers
2009-12-31 15:04:32 +00:00
def get_json(self, fields=None):
2010-09-23 16:01:48 +00:00
item = {}
2009-06-08 16:08:59 +00:00
for key in self._public_fields:
pub_key = self._public_fields.get(key, key)
if not fields or pub_key in fields:
if hasattr(self, key):
value = getattr(self, key)
else:
value = self.get(key)
if callable(value):
2010-09-23 16:01:48 +00:00
item[pub_key] = value()
2009-06-08 16:08:59 +00:00
else:
2010-09-23 16:01:48 +00:00
item[pub_key] = value
2010-09-03 13:28:44 +00:00
if not fields:
2010-09-23 16:01:48 +00:00
item['stream'] = self.get_stream()
item['poster'] = self.get_poster()
item['posters'] = self.get_posters()
if fields:
for f in fields:
if f.endswith('.length') and f[:-7] in ('cast', 'genre', 'trivia'):
2010-09-23 16:01:48 +00:00
item[f] = getattr(self.sort, f[:-7])
return item
2009-06-08 16:08:59 +00:00
2009-08-16 12:23:29 +00:00
def fields(self):
fields = {}
for f in self._meta.fields:
if f.name in self._public_fields:
fields[f.name] = {}
fields[f.name]['order'] = 'desc'
fields[f.name]['type'] = type(f)
return fields
fields = classmethod(fields)
2009-06-08 16:08:59 +00:00
def oxid(self):
return utils.oxid(self.get('title', ''), self.get('directors', []), str(self.get('year', '')),
self.get('series title', ''), self.get('episode title', ''),
self.get('season', ''), self.get('episode', ''))
2009-08-16 12:23:29 +00:00
2010-11-28 16:03:23 +00:00
def oxdb_id(self):
return utils.oxdb_id(self.get('title', ''), self.get('directors', []), str(self.get('year', '')),
self.get('season', ''), self.get('episode', ''),
self.get('episode_title', ''), self.get('episode_directors', ''), self.get('episode_year', ''))
2010-09-10 15:12:22 +00:00
'''
Search related functions
'''
def update_find(self):
2010-11-06 16:14:00 +00:00
def save(key, value):
f, created = ItemFind.objects.get_or_create(item=self, key=key)
if value not in ('', '||'):
f.value = value
f.save()
else:
f.delete()
2010-11-06 16:14:00 +00:00
save('title', '\n'.join([self.get('title'), self.get('original_title', '')]))
2010-02-22 09:25:29 +00:00
#FIXME: filter us/int title
#f.title += ' '.join([t.title for t in self.alternative_titles()])
2010-07-14 14:35:10 +00:00
2010-11-06 16:14:00 +00:00
save('year', self.get('year', ''))
2010-07-14 14:35:10 +00:00
2010-07-15 15:55:10 +00:00
for key in self.facet_keys:
2010-07-16 10:13:16 +00:00
if key == 'actor':
values = [i[0] for i in self.get('actor', [])]
elif key == 'character':
values = [i[1] for i in self.get('actor', [])]
else:
2010-08-24 17:16:33 +00:00
values = self.get(utils.plural_key(key), [])
2010-11-06 16:14:00 +00:00
save(key, '|%s|'%'|'.join(values))
save('summary', self.get('plot', '') + self.get('plot_outline', ''))
save('trivia', ' '.join(self.get('trivia', [])))
save('location', '|%s|'%'|'.join(self.get('filming_locations', [])))
2010-07-13 22:20:14 +00:00
#FIXME:
#f.dialog = 'fixme'
2010-11-28 16:03:23 +00:00
save('dialog', '\n'.join([l.value for l in Annotation.objects.filter(type='subtitle', item=self).order_by('start')]))
#FIXME: collate filenames
#f.filename = self.filename
2010-11-06 16:14:00 +00:00
all_find = ' '.join([f.value for f in ItemFind.objects.filter(item=self).exclude(key='all')])
save('all', all_find)
2009-08-16 12:23:29 +00:00
def update_sort(self):
try:
s = self.sort
2010-09-23 16:01:48 +00:00
except ItemSort.DoesNotExist:
s = ItemSort(item=self)
2009-08-16 12:23:29 +00:00
def sortNames(values):
sort_value = ''
if values:
sort_value = '; '.join([get_name_sort(name) for name in values])
2009-08-16 12:23:29 +00:00
if not sort_value:
sort_value = ''
2009-08-16 12:23:29 +00:00
return sort_value
#title
title = canonicalTitle(self.get('title'))
2010-09-03 13:28:44 +00:00
s.title = utils.sort_title(title)
2009-08-16 12:23:29 +00:00
s.country = ','.join(self.get('countries', []))
s.year = self.get('year', '')
2010-12-22 15:14:36 +00:00
s.year_desc = s.year
2009-08-16 12:23:29 +00:00
2010-07-15 15:55:10 +00:00
for key in self.person_keys:
2010-08-24 17:16:33 +00:00
setattr(s, key, sortNames(self.get(utils.plural_key(key), [])))
2010-07-14 14:35:10 +00:00
for key in ('language', 'country'):
2010-08-24 17:16:33 +00:00
setattr(s, key, ','.join(self.get(utils.plural_key(key), [])))
s.runtime = self.get('runtime', 0)
2009-08-16 12:23:29 +00:00
2010-07-16 10:13:16 +00:00
for key in ('keywords', 'genres', 'cast', 'summary', 'trivia', 'connections'):
setattr(s, key, len(self.get(key, '')))
2010-09-23 16:01:48 +00:00
s.itemId = self.itemId.replace('0x', 'xx')
s.rating = self.get('rating', -1)
s.votes = self.get('votes', -1)
2009-08-16 12:23:29 +00:00
# data from related subtitles
s.scenes = 0 #FIXME
2010-07-13 22:20:14 +00:00
s.dialog = 0 #FIXME
2009-08-16 12:23:29 +00:00
s.words = 0 #FIXME
s.wpm = 0 #FIXME
s.risk = 0 #FIXME
# data from related files
2010-12-22 15:14:36 +00:00
videos = self.main_videos()
if len(videos) > 0:
s.duration = sum([v.duration for v in videos])
s.resolution = videos[0].width * videos[0].height
s.aspectratio = int(1000 * utils.parse_decimal(v.display_aspect_ratio))
#FIXME: should be average over all files
if 'bitrate' in videos[0].info:
s.bitrate = videos[0].info['bitrate']
s.pixels = sum([v.pixels for v in videos])
s.filename = ' '.join([v.name for v in videos])
s.files = self.files.all().count()
s.size = sum([v.size for v in videos]) #FIXME: only size of movies?
2010-12-22 18:45:41 +00:00
else:
s.duration = 0
s.resolution = 0
s.aspectratio = 0
s.bitrate = 0
s.pixels = 0
s.filename = 0
s.files = 0
s.size = 0
2010-07-15 15:55:10 +00:00
for key in ('title', 'language', 'country') + self.person_keys:
setattr(s, '%s_desc'%key, getattr(s, key))
if not getattr(s, key):
setattr(s, key, u'zzzzzzzzzzzzzzzzzzzzzzzzz')
if not s.year:
s.year_desc = '';
s.year = '9999';
2010-07-13 22:20:14 +00:00
#FIXME: also deal with number based rows like genre, keywords etc
2009-08-16 12:23:29 +00:00
s.save()
2009-06-08 16:08:59 +00:00
def update_facets(self):
#FIXME: what to do with Unkown Director, Year, Country etc.
2010-07-15 15:55:10 +00:00
for key in self.facet_keys:
2010-07-16 10:13:16 +00:00
if key == 'actor':
current_values = [i[0] for i in self.get('actor', [])]
elif key == 'character':
current_values = [i[1] for i in self.get('actor', [])]
else:
2010-08-24 17:16:33 +00:00
current_values = self.get(utils.plural_key(key), [])
2010-09-23 16:01:48 +00:00
saved_values = [i.value for i in Facet.objects.filter(item=self, key=key)]
removed_values = filter(lambda x: x not in current_values, saved_values)
if removed_values:
2010-09-23 16:01:48 +00:00
Facet.objects.filter(item=self, key=key, value__in=removed_values).delete()
for value in current_values:
if value not in saved_values:
value_sort = value
2010-07-15 15:55:10 +00:00
if key in self.person_keys:
value_sort = get_name_sort(value)
f = Facet(key=key, value=value, value_sort=value_sort)
2010-09-23 16:01:48 +00:00
f.item = self
f.save()
2010-07-13 22:20:14 +00:00
year = self.get('year', None)
if year:
2010-09-23 16:01:48 +00:00
f, created = Facet.objects.get_or_create(key='year', value=year, value_sort=year, item=self)
2010-07-13 22:20:14 +00:00
else:
2010-09-23 16:01:48 +00:00
Facet.objects.filter(item=self, key='year').delete()
2010-12-04 01:26:49 +00:00
def path(self, name=''):
2010-12-05 17:51:40 +00:00
h = self.itemId
2010-12-07 19:05:59 +00:00
return os.path.join('items', h[:2], h[2:4], h[4:6], h[6:], name)
2010-12-04 01:26:49 +00:00
2010-09-10 15:12:22 +00:00
'''
Video related functions
'''
def frame(self, position, width=128):
stream = self.streams.filter(profile=settings.VIDEO_PROFILE+'.webm')[0]
2010-12-04 01:26:49 +00:00
path = os.path.join(settings.MEDIA_ROOT, self.path(), 'frames', "%d"%width, "%s.jpg"%position)
2010-09-10 15:12:22 +00:00
if not os.path.exists(path):
extract.frame(stream.video.path, path, position, width)
return path
@property
def timeline_prefix(self):
2010-12-04 01:26:49 +00:00
return os.path.join(settings.MEDIA_ROOT, self.path(), 'timeline')
2010-09-10 15:12:22 +00:00
def main_videos(self):
#FIXME: needs to check if more than one user has main files and only take from "higher" user
videos = self.files.filter(is_main=True, is_video=True, available=True)
if videos.count()>0:
first = videos[0]
user = first.instances.all()[0].volume.user
#only take videos from same user and with same width/height
def check(v):
if v.instances.filter(volume__user=user).count()>0 and \
first.width == v.width and first.height == v.height:
return True
return False
videos = filter(check, videos)
return videos
def update_streams(self):
2010-09-10 15:12:22 +00:00
files = {}
for f in self.main_videos():
2010-09-10 15:12:22 +00:00
files[utils.sort_title(f.name)] = f.video.path
#FIXME: how to detect if something changed?
if files:
2010-09-23 16:01:48 +00:00
stream, created = Stream.objects.get_or_create(item=self, profile='%s.webm' % settings.VIDEO_PROFILE)
2010-12-07 18:45:26 +00:00
stream.video.name = stream.path()
2010-09-10 15:12:22 +00:00
cmd = []
print files
2010-09-10 15:12:22 +00:00
for f in sorted(files):
cmd.append('+')
cmd.append(files[f])
if not os.path.exists(os.path.dirname(stream.video.path)):
os.makedirs(os.path.dirname(stream.video.path))
cmd = [ 'mkvmerge', '-o', stream.video.path ] + cmd[1:]
#print cmd
2010-09-10 15:12:22 +00:00
p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
#p = subprocess.Popen(cmd, stdin=subprocess.PIPE)
2010-09-10 15:12:22 +00:00
p.wait()
stream.save()
if 'video' in stream.info:
extract.timeline(stream.video.path, self.timeline_prefix)
self.stream_aspect = stream.info['video'][0]['width']/stream.info['video'][0]['height']
self.data['cuts'] = extract.cuts(self.timeline_prefix)
self.data['average_color'] = extract.average_color(self.timeline_prefix)
#extract.timeline_strip(self, self.data['cuts'], stream.info, self.timeline_prefix[:-8])
2010-09-10 15:12:22 +00:00
stream.extract_derivatives()
2010-12-01 00:00:33 +00:00
self.make_local_posters()
self.make_poster()
2010-09-10 15:12:22 +00:00
self.available = True
self.save()
'''
Poster related functions
'''
def update_poster_urls(self):
_current = {}
for s in settings.POSTER_SERVICES:
2010-09-23 16:01:48 +00:00
url = '%s?itemId=%s'%(s, self.itemId)
try:
data = json.loads(ox.net.readUrlUnicode(url))
except:
continue
for service in data:
if service not in _current:
_current[service] = []
for poster in data[service]:
_current[service].append(poster)
#FIXME: remove urls that are no longer listed
for service in _current:
for poster in _current[service]:
2010-09-23 16:01:48 +00:00
p, created = PosterUrl.objects.get_or_create(item=self, url=poster['url'], service=service)
if created:
p.width = poster['width']
p.height = poster['height']
p.save()
2010-09-10 15:12:22 +00:00
def delete_poster(self):
2010-09-13 15:25:37 +00:00
if self.poster:
path = self.poster.path
self.poster.delete()
for f in glob(path.replace('.jpg', '*.jpg')):
os.unlink(f)
def prefered_poster_url(self):
if self.poster_url:
return self.poster_url
self.update_poster_urls()
for service in settings.POSTER_PRECEDENCE:
for u in self.poster_urls.filter(service=service).order_by('-height'):
return u.url
return None
2010-12-01 00:00:33 +00:00
def make_poster(self, force=False):
2010-09-10 15:12:22 +00:00
if not self.poster or force:
url = self.prefered_poster_url()
if url:
data = ox.net.readUrl(url)
2010-09-10 15:12:22 +00:00
if force:
self.delete_poster()
self.poster.save('poster.jpg', ContentFile(data))
self.save()
else:
2010-09-10 15:12:22 +00:00
if force:
self.delete_poster()
local_posters = self.make_local_posters()
if local_posters:
with open(local_posters[0]) as f:
self.poster.save('poster.jpg', ContentFile(f.read()))
def local_posters(self):
2010-09-03 13:28:44 +00:00
part = 1
posters = {}
for f in self.main_videos():
2010-09-03 13:28:44 +00:00
for frame in f.frames.all():
2010-12-04 01:26:49 +00:00
path = self.path('poster.pandora.%s.%s.jpg'%(part, frame.position))
path = os.path.abspath(os.path.join(settings.MEDIA_ROOT, path))
posters[path] = frame.frame.path
2010-09-03 13:28:44 +00:00
part += 1
return posters
def make_local_posters(self):
posters = self.local_posters()
for poster in posters:
frame = posters[poster]
2010-12-04 01:26:49 +00:00
timeline = self.path('timeline.64.png')
timeline = os.path.abspath(os.path.join(settings.MEDIA_ROOT, timeline))
cmd = [settings.ITEM_POSTER,
'-t', self.get('title'),
'-d', ', '.join(self.get('directors', ['Unknown Director'])),
'-y', str(self.get('year', '')),
'-f', frame,
'-l', timeline,
'-p', poster
]
2010-09-23 16:01:48 +00:00
if len(self.itemId) == 7:
cmd += ['-i', self.itemId]
cmd += ['-o', self.oxdbId]
p = subprocess.Popen(cmd)
p.wait()
return posters.keys()
2010-09-03 13:28:44 +00:00
2010-09-23 16:01:48 +00:00
class ItemFind(models.Model):
2009-08-16 12:23:29 +00:00
"""
2010-11-06 16:14:00 +00:00
used to find items,
item.update_find populates this table
2010-11-06 16:14:00 +00:00
its used in manager.ItemManager
2009-08-16 12:23:29 +00:00
"""
2010-11-06 16:14:00 +00:00
class Meta:
unique_together = ("item", "key")
item = models.ForeignKey('Item', related_name='find', db_index=True)
key = models.CharField(max_length=200, db_index=True)
value = models.TextField(blank=True)
2009-06-08 16:08:59 +00:00
#FIXME: make sort based on site.json
2010-09-23 16:01:48 +00:00
class ItemSort(models.Model):
2009-08-16 12:23:29 +00:00
"""
2010-09-23 16:01:48 +00:00
used to sort items, all sort values are in here
2009-08-16 12:23:29 +00:00
"""
2010-09-23 16:01:48 +00:00
item = models.OneToOneField('Item', related_name='sort', primary_key=True)
2009-06-08 16:08:59 +00:00
2010-04-20 10:10:39 +00:00
title = models.CharField(max_length=1000, db_index=True)
director = models.TextField(blank=True, db_index=True)
country = models.TextField(blank=True, db_index=True)
year = models.CharField(max_length=4, db_index=True)
producer = models.TextField(blank=True, db_index=True)
writer = models.TextField(blank=True, db_index=True)
editor = models.TextField(blank=True, db_index=True)
cinematographer = models.TextField(blank=True, db_index=True)
language = models.TextField(blank=True, db_index=True)
runtime = models.IntegerField(blank=True, null=True, db_index=True)
keywords = models.IntegerField(blank=True, db_index=True)
genre = models.TextField(blank=True, db_index=True)
cast = models.IntegerField(blank=True, db_index=True)
summary = models.IntegerField(blank=True, db_index=True)
trivia = models.IntegerField(blank=True, db_index=True)
connections = models.IntegerField(blank=True, db_index=True)
rating = models.FloatField(blank=True, db_index=True)
votes = models.IntegerField(blank=True, db_index=True)
scenes = models.IntegerField(blank=True, db_index=True)
2010-07-13 22:20:14 +00:00
dialog = models.IntegerField(null=True, blank=True, db_index=True)
2010-04-20 10:10:39 +00:00
words = models.IntegerField(null=True, blank=True, db_index=True)
wpm = models.IntegerField('Words per Minute', null=True, blank=True, db_index=True)
risk = models.IntegerField(null=True, blank=True, db_index=True)
2010-09-23 16:01:48 +00:00
itemId = models.CharField('ID', max_length=128, blank=True, db_index=True)
2010-04-20 10:10:39 +00:00
duration = models.FloatField(default=-1, db_index=True)
2010-12-22 15:14:36 +00:00
resolution = models.BigIntegerField(blank=True, db_index=True)
2010-04-20 10:10:39 +00:00
aspectratio = models.IntegerField('Aspect Ratio', blank=True, db_index=True)
bitrate = models.IntegerField(blank=True, db_index=True)
pixels = models.BigIntegerField(blank=True, db_index=True)
2010-12-23 07:55:44 +00:00
filename = models.CharField(max_length=1024, blank=True, db_index=True)
2010-04-20 10:10:39 +00:00
files = models.IntegerField(blank=True, db_index=True)
size = models.BigIntegerField(blank=True, db_index=True)
2009-06-08 16:08:59 +00:00
#required to move empty values to the bottom for both asc and desc sort
title_desc = models.CharField(max_length=1000, db_index=True)
director_desc = models.TextField(blank=True, db_index=True)
country_desc = models.TextField(blank=True, db_index=True)
year_desc = models.CharField(max_length=4, db_index=True)
producer_desc = models.TextField(blank=True, db_index=True)
writer_desc = models.TextField(blank=True, db_index=True)
editor_desc = models.TextField(blank=True, db_index=True)
cinematographer_desc = models.TextField(blank=True, db_index=True)
language_desc = models.TextField(blank=True, db_index=True)
2010-09-23 16:01:48 +00:00
_private_fields = ('id', 'item')
2009-12-31 15:04:32 +00:00
#return available sort fields
#FIXME: should return mapping name -> verbose_name
def fields(self):
fields = []
2009-06-08 16:08:59 +00:00
for f in self._meta.fields:
if f.name not in self._private_fields:
2009-12-31 15:04:32 +00:00
name = f.verbose_name
name = name[0].capitalize() + name[1:]
fields.append(name)
return tuple(fields)
fields = classmethod(fields)
2009-06-08 16:08:59 +00:00
class Facet(models.Model):
2010-09-23 16:01:48 +00:00
item = models.ForeignKey('Item', related_name='facets')
key = models.CharField(max_length=200, db_index=True)
value = models.CharField(max_length=200)
value_sort = models.CharField(max_length=200)
2009-06-08 16:08:59 +00:00
def save(self, *args, **kwargs):
if not self.value_sort:
self.value_sort = self.value
super(Facet, self).save(*args, **kwargs)
2009-08-16 12:23:29 +00:00
2010-09-03 13:28:44 +00:00
class Stream(models.Model):
class Meta:
2010-09-23 16:01:48 +00:00
unique_together = ("item", "profile")
2010-09-03 13:28:44 +00:00
2010-09-23 16:01:48 +00:00
item = models.ForeignKey(Item, related_name='streams')
2010-09-03 13:28:44 +00:00
profile = models.CharField(max_length=255, default='96p.webm')
2010-12-04 01:26:49 +00:00
video = models.FileField(default=None, blank=True, upload_to=lambda f, x: f.path())
2010-09-03 13:28:44 +00:00
source = models.ForeignKey('Stream', related_name='derivatives', default=None, null=True)
available = models.BooleanField(default=False)
info = fields.DictField(default={})
def __unicode__(self):
return u"%s/%s" % (self.item, self.profile)
2010-09-03 13:28:44 +00:00
2010-12-04 01:26:49 +00:00
def path(self):
2010-12-05 17:51:40 +00:00
return self.item.path(self.profile)
2010-12-04 01:26:49 +00:00
2010-09-03 13:28:44 +00:00
def extract_derivatives(self):
if settings.VIDEO_H264:
profile = self.profile.replace('.webm', '.mp4')
2010-09-23 16:01:48 +00:00
derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
2010-09-15 13:03:00 +00:00
if created:
derivative.source = self
2010-09-03 13:28:44 +00:00
derivative.video.name = self.video.name.replace(self.profile, profile)
derivative.encode()
2010-09-15 13:03:00 +00:00
derivative.save()
2010-09-03 13:28:44 +00:00
for p in settings.VIDEO_DERIVATIVES:
profile = p + '.webm'
target = self.video.path.replace(self.profile, profile)
2010-09-23 16:01:48 +00:00
derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
2010-09-15 13:03:00 +00:00
if created:
derivative.source = self
2010-09-03 13:28:44 +00:00
derivative.video.name = self.video.name.replace(self.profile, profile)
derivative.encode()
2010-09-15 13:03:00 +00:00
derivative.save()
2010-09-03 13:28:44 +00:00
if settings.VIDEO_H264:
profile = p + '.mp4'
2010-09-23 16:01:48 +00:00
derivative, created = Stream.objects.get_or_create(profile=profile, item=self.item)
2010-09-15 13:03:00 +00:00
if created:
derivative.source = self
2010-09-03 13:28:44 +00:00
derivative.video.name = self.video.name.replace(self.profile, profile)
derivative.encode()
2010-09-15 13:03:00 +00:00
derivative.save()
2010-09-03 13:28:44 +00:00
return True
def encode(self):
if self.source:
video = self.source.video.path
target = self.video.path
profile = self.profile
info = ox.avinfo(video)
if extract.stream(video, target, profile, info):
self.available=True
self.save()
def save(self, *args, **kwargs):
if self.video and not self.info:
self.info = ox.avinfo(self.video.path)
super(Stream, self).save(*args, **kwargs)
class PosterUrl(models.Model):
class Meta:
2010-09-23 16:01:48 +00:00
unique_together = ("item", "service", "url")
2010-09-13 15:19:38 +00:00
ordering = ('-height', )
2010-09-23 16:01:48 +00:00
item = models.ForeignKey(Item, related_name='poster_urls')
url = models.CharField(max_length=1024)
service = models.CharField(max_length=1024)
width = models.IntegerField(default=80)
height = models.IntegerField(default=128)
def __unicode__(self):
2010-09-23 16:01:48 +00:00
return u'%s %s %dx%d' % (unicode(self.item), self.service, self.width, self.height)