commit stuff

backend/load.py

@@ -59,11 +59,19 @@ def loadIMDb(imdbId):
         setattr(movie, key, business[key])
 
     movie.save()
+    models.AlternativeTitle.objects.filter(movie=movie).delete()
+    for i in oxweb.imdb.getMovieAKATitles(imdbId):
+        t = models.AlternativeTitle()
+        t.movie = movie
+        t.title = i[0]
+        t.type = i[1]
+        t.save()
 
     #FIXME: related tables should be cleaned to not accumulate cruft
     #Country
     models.MovieCountry.objects.filter(movie=movie).delete()
     position = 0
+    if 'country' in info:
         for i in info['country']:
             debug("add country", i)
             country = models.Country.get_or_create(i)
@@ -73,6 +81,7 @@ def loadIMDb(imdbId):
     #Language
     models.MovieLanguage.objects.filter(movie=movie).delete()
     position = 0
+    if 'language' in info:
         for i in info['language']:
             debug("add language", i)
             language = models.Language.get_or_create(i)
@@ -89,6 +98,7 @@ def loadIMDb(imdbId):
 
     #Genre
     movie.genres.all().delete()
+    if 'genre' in info:
         for i in info['genre']:
             debug("add genre", i)
             genre = models.Genre.get_or_create(i)
@@ -106,7 +116,7 @@ def loadIMDb(imdbId):
     position = 0
     trivia = oxweb.imdb.getMovieTrivia(imdbId)
     for i in trivia:
-        debug("add trivia", g)
+        debug("add trivia", i)
         t = models.Trivia()
         t.movie = movie
         t.trivia = i
@@ -127,8 +137,16 @@ def loadIMDb(imdbId):
             models.Cast.link(movie, person, role, character, position)
             position += 1
 
-    #FIXME: connections
+    movie.connections.all().delete()
-    #m.addMovieConnections(IMDb['connections'])
+    connections = oxweb.imdb.getMovieConnections(imdbId)
+    for relation in connections:
+        for otherId in connections[relation]:
+            try:
+                object = models.Movie.objects.get(imdbId=otherId)
+                debug("add connection", relation, object)
+                models.Connection.get_or_create(movie, relation, object)
+            except models.Movie.DoesNotExist:
+                pass
 
     reviews = oxweb.imdb.getMovieExternalReviews(imdbId)
     for r in reviews:

backend/managers.py

@@ -1,29 +1,136 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+import re
+from datetime import datetime
+from urllib2 import unquote
+
 from django.contrib.auth.models import User
 from django.core.exceptions import ObjectDoesNotExist
-from django.db import models
+from django.db.models import Q, Manager
-from django.db.models import Q
 
+import models
 
-class MovieManager(models.Manager):
+def keyType(key):
+    if key in ('released'):
+        return "date"
+    if key in ('year', 'cast.length'):
+        return "int"
+    if key in ('rating', 'votes'):
+        return "float"
+    return "string"
+
+class MovieManager(Manager):
     def get_query_set(self):
         return super(MovieManager, self).get_query_set()
 
-    def find(self, q="", f="all", s="title", a="desc", l="all", o=0, n=100, p=None):
+    def find(self, request):
-        qs = self.get_query_set()
+        '''
-
+            construct query set from q value in request,
-        if q:
+            also checks for lists.
-            if f == "all":
+            range and order must be applied later
-                qs = qs.filter(title__icontains=q)
+        '''
+        for i in request.META['QUERY_STRING'].split('&'):
+          if i.startswith('q='):
+            q = i[2:]
+        op = ','
+        if '|' in q:
+            op = '|'
+        conditions = []
+        for e in q.split(op):
+            e = e.split(':')
+            if len(e) == 1: e = ['all'] + e
+            k, v = e
+            exclude = False
+            if v.startswith('!'):
+                v = v[1:]
+                exclude = True
+            if keyType(k) == "string":
+                startswith = v.startswith('^')
+                endswith = v.endswith('$')
+                if startswith and endswith:
+                    v = v[1:-1]
+                    k = '%s__iexact' % k
+                elif startswith:
+                    v = v[1:]
+                    k = '%s__istartswith' % k
+                elif v.endswith('$'):
+                    v = v[:-1]
+                    k = '%s__iendswith' % k
                 else:
-                field = str("find__%s__icontains"%f)
+                    k = '%s__icontains' % k
-                qs = qs.filter(**{field: q})
+                k = 'find__%s' % k
+                v = unquote(v)
+                if exclude:
+                    conditions.append(~Q(**{k:v}))
+                else:
+                    conditions.append(Q(**{k:v}))
+            else:
+                def parseDate(d):
+                    while len(d) < 3:
+                        d.append(1)
+                    return datetime(*[int(i) for i in d])
+                #1960-1970
+                match = re.compile("(-?[\d\.]+?)-(-?[\d\.]+$)").findall(v)
+                if match:
+                    v1 = match[0][0]
+                    v2 = match[0][1]
+                    if keyType(k) == "date":
+                        v1 = parseDate(v1.split('.'))
+                        v2 = parseDate(v2.split('.'))
+                    if exclude: #!1960-1970
+                        k1 = str('%s__lt' % k)
+                        k2 = str('%s__gte' % k)
+                        conditions.append(Q(**{k1:v1})|Q(**{k2:v2}))
+                    else: #1960-1970
+                        k1 = str('%s__gte' % k)
+                        k2 = str('%s__lt' % k)
+                        conditions.append(Q(**{k1:v1})&Q(**{k2:v2}))
+                else:
+                    if keyType(k) == "date":
+                        v = parseDate(v.split('.'))
+                    k = str('%s' % k)
+                    if exclude: #!1960
+                        conditions.append(~Q(**{k:v}))
+                    else: #1960
+                        conditions.append(Q(**{k:v}))
 
-        order_by = s
+        #join query with operator
-        if a == "desc":
+        qs = self.get_query_set()
-          order_by = "-sort__" + order_by
+        if conditions:
-        qs = qs.order_by(order_by)
+            q = conditions[0]
+            for c in conditions[1:]:
+                if op == '|':
+                    q = q | c
+                else:
+                    q = q & c
+            qs = qs.filter(q)
 
-        return qs[o:n]
+        # filter list, works for own or public lists
+        l = request.GET.get('l', 'all')
+        if l != "all":
+            l = l.split(":")
+            only_public = True
+            if not request.user.is_anonymous():
+                if len(l) == 1: l = [request.user.username] + l
+                if request.user.username == l[0]:
+                    only_public = False
+            if len(l) == 2:
+                lqs = models.List.objects.filter(name=l[1], user__username=l[0])
+                if only_public:
+                    lqs = qls.filter(public=True)
+                if lqs.count() == 1:
+                    qs = qs.filter(listitem__list__id=lqs[0].id)
+        return qs
+
+class ArchiveFileManager(Manager):
+    def get_query_set(self):
+        return super(UserFileManager, self).get_query_set()
+
+    def by_oshash(self, oshash):
+        q = self.get_query_set()
+        q.filter(movie_file__oshash=oshash)
+        if q.count() == 0:
+            raise models.UserFile.DoesNotExist("%s matching oshash %s does not exist." %
+                 (models.UserFile._meta.object_name, oshash))
+        return q[0]
 

backend/models.py

@@ -1,11 +1,14 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
+import re
 import os.path
 from django.db import models
 from django.db.models import Q
 from django.contrib.auth.models import User
 
 import oxlib
+from oxlib import stripTags
+from oxlib.normalize import canonicalTitle, canonicalName
 
 import utils
 import managers
@@ -61,11 +64,19 @@ class Movie(models.Model):
     #FIXME: include role and character
     def cast(self):
         cast = []
-        for c in Cast.objects.filter(movie=self, role=cast).order_by('position'):
+        for c in Cast.objects.filter(movie=self, role='cast').order_by('position'):
             cast.append((c.person.name, c.character))
         return tuple(cast)
         #return self.person.filter(cast__role='cast').order_by('cast__position')
 
+    def connections_json(self):
+        connections = {}
+        for connection in self.connections.all():
+            if connection.relation not in connections:
+                connections[connection.relation] = []
+            connections[connection.relation].append(connection.object.movieId)
+        return connections
+
     def filtered_reviews(self):
         whitelist = ReviewWhitelist.objects.all()
         q = Q(id=-1)
@@ -73,22 +84,14 @@ class Movie(models.Model):
             q = q | Q(url__contains=w.url)
         return self.reviews.filter(q)
 
-    risk = models.IntegerField(null=True, blank=True)
+    rights_level = models.IntegerField(default=-1)
-    rights_level = models.IntegerField(null=True, blank=True)
-    rights_text = models.TextField(blank=True)
    
-    #title_english = models.TextField(blank=True)
+    #FIXME: use data.0xdb.org
-    #FIXME: join AltTitle
-
-    #FIXME: use mapping
     tpb_id = models.CharField(max_length=128, blank=True)
     kg_id = models.CharField(max_length=128, blank=True)
     open_subtitle_id = models.IntegerField(null=True, blank=True)
     wikipedia_url = models.TextField(blank=True)
 
-    #FIXME: join Location
-    #locations = models.TextField(blank=True)
-
     #Series information
     series_imdb = models.CharField(max_length=7, default='')
     series_title = models.TextField(blank=True, default='')
@@ -107,7 +110,7 @@ class Movie(models.Model):
     scene_height = models.IntegerField(null=True, blank=True)
 
     def __unicode__(self):
-        return "%s (%s)" % (self.title, self.year)
+        return u'%s (%s)' % (self.title, self.year)
 
     def save(self, *args, **kwargs):
         if self.imdbId:
@@ -115,9 +118,10 @@ class Movie(models.Model):
         else:
             mid = self.oxdbId
         self.movieId = mid
-        #FIXME: update sort and find values here
         
         super(Movie, self).save(*args, **kwargs)
+        self.updateFind()
+        self.updateSort()
 
     _public_fields = {
         'movieId': 'id',
@@ -138,6 +142,8 @@ class Movie(models.Model):
         'episode': 'episode',
         'filtered_reviews': 'reviews',
         'trivia': 'trivia',
+        'alternative_titles': 'alternative_titles',
+        'connections': 'connections_json'
     }
     def json(self, fields=None):
         movie = {}
@@ -147,12 +153,25 @@ class Movie(models.Model):
                 value = getattr(self, key)
                 if key in ('directors', 'writers', 'filtered_reviews'):
                     movie[pub_key] = tuple([v.json() for v in value()])
-                elif key in ('countries', 'keywords', 'genres', 'trivia'):
+                elif key in ('countries', 'keywords', 'genres', 'trivia', 'alternative_titles'):
                     movie[pub_key] = tuple([v.json() for v in value.all()])
                 else:
                     movie[pub_key] = value
+        for f in fields:
+            if f.endswith('.length') and f[:-7] in ('cast', 'genre', 'trivia'):
+                movie[f] = getattr(self.sort.all()[0], f[:-7])
         return movie
 
+    def fields(self):
+        fields = {}
+        for f in self._meta.fields:
+            if f.name in self._public_fields:
+                fields[f.name] = {}
+                fields[f.name]['order'] = 'desc'
+                fields[f.name]['type'] = type(f)
+        return fields
+    fields = classmethod(fields)
+
     #Class functions to get Movies by ID, right now movieId, imdbId and oxdbId
     #FIXME: this should go into a manager
     def byMovieId(self, movieId):
@@ -162,29 +181,124 @@ class Movie(models.Model):
     byMovieId = classmethod(byMovieId)
 
     def byImdbId(self, imdbId):
-        q = self.objects.filter(imdbId=imdbId)
+        return self.objects.get(imdbId=imdbId)
-        if q.count() == 0:
-            raise self.DoesNotExist("%s matching imdb id %s does not exist." % (self._meta.object_name, imdbId))
-        return q[0]
     byImdbId = classmethod(byImdbId)
 
     def byOxdbId(self, oxdbId):
-        q = self.objects.filter(oxdbId=oxdbId)
+        return self.objects.get(oxdbId=oxdbId)
-        if q.count() == 0:
-            raise self.DoesNotExist("%s matching oxdb id %s does not exist." % (self._meta.object_name, oxdbId))
-        return q[0]
     byOxdbId = classmethod(byOxdbId)
 
     def oxid(self):
-        directors = ",".join([d.name for d in self.directors()])
+        directors = ','.join([d.name for d in self.directors()])
-        return utils.oxid(self.title, directors, self.year, self.series_title, self.episode_title, self.season, self.episode)
+        return utils.oxid(self.title, directors, self.year,
+                          self.series_title, self.episode_title, self.season, self.episode)
+
+    def updateFind(self):
+        if self.find.count() == 0:
+            f = MovieFind()
+            f.movie = self
+        else:
+            f = self.find.all()[0]
+        f.title = self.title + ' '.join([t.title for t in self.alternative_titles.all()])
+        f.director = ' '.join([i.name for i in self.directors()])
+        f.country = ' '.join([i.name for i in self.countries.all()])
+        f.year = self.year
+        f.language = ' '.join([i.name for i in self.languages.all()])
+        f.writer = ' '.join([i.name for i in self.writers()])
+        f.producer = ' '.join([i.name for i in self.producers()])
+        f.editor = ' '.join([i.name for i in self.editors()])
+        f.cinematographer = ' '.join([i.name for i in self.cinematographers()])
+        f.cast = ' '.join(['%s %s' % i for i in self.cast()])
+        f.genre = ' '.join([i.name for i in self.genres.all()])
+        f.keywords = ' '.join([i.name for i in self.keywords.all()])
+        f.summary = self.plot + self.plot_outline
+        f.trivia = ' '.join([i.trivia for i in self.trivia.all()])
+        f.location = ' '.join([i.name for i in self.locations.all()])
+        f.filename = self.filename
+        f.all = ' '.join([f.title, f.director, f.country, f.year, f.language,
+                          f.writer, f.producer, f.editor, f.cinematographer,
+                          f.cast, f.genre, f.keywords, f.summary, f.trivia,
+                          f.location, f.filename])
+        f.save()
+
+    def updateSort(self):
+        if self.sort.count() == 0:
+            s = MovieSort()
+            s.movie = self
+        else:
+            s = self.sort.all()[0]
+
+        def sortName(value):
+            sort_value = '~'
+            if value:
+                sort_value = stripTags(value).split(',')
+                sort_value = '; '.join([canonicalName(name.strip()) for name in sort_value])
+                sort_value = sort_value.replace(u'\xc5k', 'A')
+            if not sort_value:
+                sort_value = '~'
+            return sort_value
+
+        #title
+        title = canonicalTitle(self.title)
+        title = re.sub(u'[\'!¿¡,\.;\-"\:\*\[\]]', '', title)
+        title = title.replace(u'Æ', 'Ae')
+        #pad numbered titles
+        numbers = re.compile('^(\d+)').findall(title)
+        if numbers:
+            padded = '%010d' % int(numbers[0])
+            title = re.sub('^(\d+)', padded, title)
+
+        s.title = title.strip()
+
+        directors = ','.join([i.name for i in self.directors()])
+        s.director = sortName(directors)
+
+        s.country = ','.join([i.name for i in self.countries.all()])
+        s.year = self.year
+
+        names = ','.join([i.name for i in self.producers()])
+        s.producer = sortName(names)
+        names = ','.join([i.name for i in self.writers()])
+        s.writer = sortName(names)
+        names = ','.join([i.name for i in self.editors()])
+        s.editor = sortName(names)
+        names = ','.join([i.name for i in self.cinematographers()])
+        s.cinematographer = sortName(names)
+
+        s.country = ','.join([i.name for i in self.languages.all()])
+        s.runtime = self.runtime
+
+        s.keywords = self.keywords.all().count()
+        s.genre = self.genres.all().count()
+        s.cast = len(self.cast())
+        s.summary = len(self.plot.split())
+        s.trivia = self.trivia.all().count()
+        s.connections = self.connections.all().count()
+        s.movieId = self.movieId
+
+        # data from related subtitles
+        s.scenes = 0 #FIXME
+        s.words =  0 #FIXME
+        s.wpm =    0 #FIXME
+        s.risk =   0 #FIXME
+        # data from related files
+        s.duration = 0 #FIXME
+        s.resolution = 0 #FIXME
+        s.aspectratio = 0 #FIXME
+        s.bitrate = 0 #FIXME
+        s.pixels = 0 #FIXME
+        s.filename = 0 #FIXME
+        s.files = 0 #FIXME
+        s.size = 0 #FIXME
+        s.save()
 
-'''
-    used to search movies, all search values are in here
-'''
 class MovieFind(models.Model):
-    movie = models.ForeignKey('Movie', related_name='find')
+    """
+        used to search movies, all search values are in here
+    """
+    movie = models.ForeignKey('Movie', related_name='find', unique=True)
 
+    all = models.TextField(blank=True)
     title = models.CharField(max_length=1000)
     director = models.TextField(blank=True)
     country = models.TextField(blank=True)
@@ -193,8 +307,8 @@ class MovieFind(models.Model):
     writer = models.TextField(blank=True)
     producer = models.TextField(blank=True)
     editor = models.TextField(blank=True)
-    cinematographers = models.TextField(blank=True)
+    cinematographer = models.TextField(blank=True)
-    cast = models.IntegerField(blank=True)
+    cast = models.TextField(blank=True)
     #person
 
     genre = models.TextField(blank=True)
@@ -202,7 +316,6 @@ class MovieFind(models.Model):
     summary = models.TextField(blank=True)
     trivia = models.TextField(blank=True)
     locations = models.TextField(blank=True)
-    connections = models.TextField(blank=True)
 
     #only for own files or as admin?
     filename = models.TextField(blank=True)
@@ -221,11 +334,11 @@ class MovieFind(models.Model):
         return tuple(options)
     options = classmethod(options)
 
-'''
-    used to sort movies, all sort values are in here
-'''
 class MovieSort(models.Model):
-    movie = models.ForeignKey('Movie', related_name='sort')
+    """
+        used to sort movies, all sort values are in here
+    """
+    movie = models.ForeignKey('Movie', related_name='sort', unique=True)
 
     title = models.CharField(max_length=1000)
     director = models.TextField(blank=True)
@@ -235,10 +348,10 @@ class MovieSort(models.Model):
     producer = models.TextField(blank=True)
     writer = models.TextField(blank=True)
     editor = models.TextField(blank=True)
-    cinematographers = models.TextField(blank=True)
+    cinematographer = models.TextField(blank=True)
 
     language = models.TextField(blank=True)
-    runtime = models.IntegerField(blank=True)
+    runtime = models.IntegerField(blank=True, null=True)
 
     keywords = models.IntegerField(blank=True)
     genre = models.TextField(blank=True)
@@ -279,8 +392,8 @@ class MovieSort(models.Model):
 
 class AlternativeTitle(models.Model):
     movie = models.ForeignKey(Movie, related_name='alternative_titles')
-    type = models.CharField(max_length=128)
     title = models.TextField()
+    type = models.CharField(max_length=1000)
 
     class Meta:
         ordering = ('title', )
@@ -288,6 +401,9 @@ class AlternativeTitle(models.Model):
     def __unicode__(self):
         return self.title
 
+    def json(self):
+        return (self.title, self.type)
+
 def get_or_create(model, name):
     try:
         o = model.objects.get(name=name)
@@ -408,10 +524,9 @@ class MovieCountry(models.Model):
         return link
     link = classmethod(link)
 
-
 class Language(models.Model):
     name = models.CharField(max_length=200, unique=True)
-    movies = models.ManyToManyField(Movie, related_name='language', through="MovieLanguage")
+    movies = models.ManyToManyField(Movie, related_name='languages', through="MovieLanguage")
 
     class Meta:
         ordering = ('name', )
@@ -464,7 +579,6 @@ class Keyword(models.Model):
     def json(self):
         return self.name
 
-
 class Genre(models.Model):
     name = models.CharField(max_length=200, unique=True)
     movies = models.ManyToManyField(Movie, related_name='genres')
@@ -485,6 +599,14 @@ class Location(models.Model):
     movies = models.ManyToManyField(Movie, related_name='locations')
     #fixme: geo data
 
+    lat_sw = models.FloatField(default=0)
+    lng_sw = models.FloatField(default=0)
+    lat_ne = models.FloatField(default=0)
+    lng_ne = models.FloatField(default=0)
+    lat_center = models.FloatField(default=0)
+    lng_center = models.FloatField(default=0)
+    area = models.FloatField(default=-1)
+
     class Meta:
         ordering = ('name', )
 
@@ -508,17 +630,57 @@ class Trivia(models.Model):
         return self.trivia
 
     def json(self):
-        return self.trivia
+        trivia = self.trivia
+        trivia = oxlib.fixAmpersands(trivia)
+        trivia = re.sub('<a href="(/name/nm.*?)">(.*?)</a>', '<a href="/?f=name&amp;q=\\2">\\2</a>', trivia)
+        trivia = re.sub('<a href="/title/tt(.*?)/">(.*?)</a>', '<a href="/\\1">\\2</a>', trivia)
+        return trivia
 
-class MovieFile(models.Model):
+class Connection(models.Model):
+    subject = models.ForeignKey(Movie, related_name='connections')
+    relation = models.CharField(max_length=512)
+    object = models.ForeignKey(Movie)
+
+    def get_or_create(model, subject, relation, object, reverse=True):
+        q = model.objects.filter(subject=subject, relation=relation, object=object)
+        if q.count() > 0:
+            o = q[0]
+        else:
+            o = model.objects.create(subject=subject, relation=relation, object=object)
+            o.save()
+        if reverse:
+            _map = {
+                'Edited into': 'Edited from',
+                'Features': 'Featured in',
+                'Follows': 'Followed by',
+                'References': 'Referenced in',
+                'Remake of': 'Remade as',
+                'Spin off from': 'Spin off',
+                'Spoofs': 'Spoofed in',
+                'Version of': 'Version of',
+            }
+            if relation in _map.values():
+                for k in _map:
+                    if _map[k] == relation:
+                        reverse_relation = k
+            else:
+                reverse_relation = _map[relation]
+            o2 = model.get_or_create(object, reverse_relation, subject, reverse=False)
+        return o
+    get_or_create = classmethod(get_or_create)
+
+    def __unicode__(self):
+        return '%s %s %s' % (self.subject, self.relation, self.object)
+
+class File(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
 
     oshash = models.CharField(blank=True, unique=True, max_length=16)
-    sha1hash = models.CharField(blank=True, unique=True, max_length=40)
+    sha1 = models.CharField(blank=True, unique=True, max_length=40)
-    md5sum = models.CharField(blank=True, unique=True, max_length=32)
+    md5 = models.CharField(blank=True, unique=True, max_length=32)
 
-    movie = models.ForeignKey('Movie', related_name="files")
+    movie = models.ForeignKey('Movie', related_name="files", default=None)
 
     computed_path = models.CharField(blank=True, max_length=2048)
     size = models.IntegerField(default=-1)
@@ -540,35 +702,87 @@ class MovieFile(models.Model):
     bpp = models.FloatField(default=-1)
     pixels = models.IntegerField(default=0)
 
-    part = models.IntegerField(default=1)
+    part = models.IntegerField(default=0)
+
+    def get_or_create(model, oshash):
+        try:
+            f = model.objects.get(oshash=oshash)
+        except model.DoesNotExist:
+            f = model.objects.create(oshash=oshash)
+            f.save()
+        return f
+    get_or_create = classmethod(get_or_create)
 
     def __unicode__(self):
         return "%s (%s)" % (self.computed_path, self.oshash)
 
-class UserFile(models.Model):
+class Archive(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
-    user = models.ForeignKey(User)
+    name = models.CharField(max_length=512, unique=True)
-    movie_file = models.ForeignKey(MovieFile)
+    users = models.ManyToManyField(User, related_name='archives')
+
+class ArchiveFile(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+    archive = models.ForeignKey(Archive)
+    movie_file = models.ForeignKey(File)
     path = models.CharField(blank=True, max_length=2048)
 
+    objects = managers.ArchiveFileManager()
+    
+    def update(self, data):
+        """
+            only add, do not overwrite keys in movie_file
+        """
+        for key in ('duration', 'video_codec', 'pixel_format', 'width', 'height',
+                    'pixel_aspect_ratio', 'display_aspect_ratio', 'framerate',
+                    'audio_codec', 'samplerate', 'channels', 'size', 'sha1', 'md5'):
+            if key in data and not getattr(self.movie_file, key):
+                setattr(self.movie_file, key, data[key])
+        self.path = data.get('path', '')
+        self.movie_file.save()
+        self.save()
+
+    def get_or_create(model, archive, oshash):
+        try:
+            f = model.objects.by_oshash(oshash=oshash)
+        except model.DoesNotExist:
+            f = model.objects.create()
+            f.movie_file = File.get_or_create(oshash)
+            f.archive = archive
+            f.save()
+        return f
+    get_or_create = classmethod(get_or_create)
+
     def __unicode__(self):
-        return "%s (%s)" % (self.path, unicode(self.user))
+        return '%s (%s)' % (self.path, unicode(self.user))
 
 class Subtitle(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
     user = models.ForeignKey(User)
 
-    movie_file = models.ForeignKey(MovieFile)
+    movie_file = models.ForeignKey(File)
     language = models.CharField(max_length=16)
     srt = models.TextField(blank=True)
 
+    def get_or_create(model, user, oshash, language):
+        q = model.objects.filter(movie_file__oshash=oshash, language=language, user=user)
+        if q.count() > 0:
+            s = q[0]
+        else:
+            movie_file = models.File.get_or_create(oshash=oshash)
+            s = model.objects.create(user=user, language=language, movie_file=movie_file)
+            s.save()
+        return s
+    get_or_create = classmethod(get_or_create)
+
     def __unicode__(self):
-        return "%s.%s.srt" % (os.path.splitext(self.movie_file.computed_path)[0], self.language)
+        return '%s.%s.srt' % (os.path.splitext(self.movie_file.computed_path)[0], self.language)
 
 class Review(models.Model):
-    movie = models.ForeignKey('Movie', related_name="reviews")
+    movie = models.ForeignKey('Movie', related_name='reviews')
     title = models.CharField(blank=True, max_length=2048)
     url = models.CharField(blank=True, max_length=2048)
 
@@ -602,7 +816,8 @@ class List(models.Model):
     created = models.DateTimeField(auto_now_add=True)
     modified = models.DateTimeField(auto_now=True)
     user = models.ForeignKey(User)
-    title = models.CharField(max_length=255, unique=True)
+    name = models.CharField(max_length=255, unique=True)
+    public = models.BooleanField(default=False)
     movies = models.ManyToManyField(Movie, related_name='lists', through='ListItem')
 
     def add(self, movie):
@@ -617,7 +832,7 @@ class List(models.Model):
         self.ListItem.objects.all().filter(movie=movie, list=self).delete()
 
     def __unicode__(self):
-        return u"%s (%s)" % (self.title, unicode(self.user))
+        return u'%s (%s)' % (self.title, unicode(self.user))
 
 class ListItem(models.Model):
     created = models.DateTimeField(auto_now_add=True)
@@ -626,5 +841,5 @@ class ListItem(models.Model):
     movie = models.ForeignKey(Movie)
 
     def __unicode__(self):
-        return u"%s in %s" % (unicode(self.movie), unicode(self.list))
+        return u'%s in %s' % (unicode(self.movie), unicode(self.list))
 

backend/urls.py

@@ -24,9 +24,10 @@ urlpatterns = patterns("oxdb.backend.views",
     (r'^find', 'find'),
     (r'^files/find', 'find_files'),
     (r'^files/info', 'file_info'),
-    (r'^files/add', 'add_file'),
+    (r'^files/(?P<archive>.+)/add', 'add_file'),
-    (r'^files/remove', 'remove_file'),
+    (r'^files/(?P<archive>.+)/remove', 'remove_file'),
-    (r'^subtitle/get', 'get_subtitle'),
+    (r'^subtitle/get', 'subtitles'),
+    (r'^preferences', 'preferences'),
 
     # Example:
     # (r'^oxdata/', include('oxdata.foo.urls')),

backend/views.py

@@ -1,19 +1,69 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=4:sts=4:ts=4
 import os.path
+import re
+from datetime import datetime
+from urllib2 import unquote
+
 from django.db import models
 from django.db.models import Q, Avg, Count
 from django.contrib.auth.models import User
 from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
 from django.template import RequestContext
 from django.core.paginator import Paginator
+from django.contrib.auth.decorators import login_required
 
 from oxdb.utils.shortcuts import render_to_json_response
 
 import models
 
+    
 '''
-.length -> _sort
+
+field.length -> movie.sort.all()[0].field
+o=0&n=100
+
+
+a & b  | c & d
+
+query
+
+
+l=user:name or l=name
+q=year:1980,hello,country:usa
+q=year:1980,hello,country:!usa
+q=title:^the$
+q=title:^100%24$
+q=year:<1970,year:>1960
+q=year:<1960,year:>1950,title:sex
+
+!1960-1970
+2009.08.02.22.26.35-2009.08.02.22.26.35
+
+!^the
+
+ (dddd.dd.dd)-(dddd.dd.dd)
+
+5-8
+10000-20000
+
+<2009-08-02-22-26-35
+
+>2009-08-02-22-26-35
+2009-08-02-22-26-35<
+
+^the the*
+*foo foo$
+*foo* foo
+
+s=director:asc,year:desc               default: director:asc,year:desc
+r=0:100 or r=100 or r=100:             default: 0:100
+k=id,title,director,date,cast.length   default: title,director,year,country
+
+
+id=0133093
+
+/json/find?l=all&s=date&f=all&q=&a=desc&p=id,title,director,date,cast.length
 
 /json/find?o=0&n=100&l=all&s=date&f=all&q=&a=desc&p=id,title,director,date,cast.length
     {
@@ -56,20 +106,32 @@ import models
 #auto compleat in find box
 
 '''
-def parse_query(get):
+
+def order_query(qs, s, prefix='sort__'):
+    order_by = []
+    for e in s.split(','):
+        o = e.split(':')
+        if len(o) == 1: o.append('asc')
+        order = '%s%s' % (prefix, o[0])
+        if o[1] == 'desc':
+           order = '-%s' % order
+        order_by.append(order)
+    if order_by:
+        qs = qs.order_by(*order_by)
+    return qs
+
+def parse_query(request):
+    get = request.GET
     query = {}
-    query["o"] = 0
+    query['i'] = 0
-    query["n"] = 100
+    query['o'] = 100
-    query["q"] = "The"
+    query['s'] = 'title:asc'
-    query["f"] = "all"
-    query["s"] = "title"
-    query["a"] = "desc"
     def parse_dict(s):
         d = s.split(",")
         return [i.strip() for i in d]
-    _dicts = ['p', ]
+    _dicts = ['k', ]
     _ints = ['o', 'n']
-    for key in ('q', 'f', 's', 'a', 'p', 'g', 'o', 'n'):
+    for key in ('s', 'k', 'g', 'l'):
         if key in get:
             if key in _ints:
                 query[key] = int(get[key])
@@ -77,44 +139,57 @@ def parse_query(get):
                 query[key] = parse_dict(get[key])
             else:
                 query[key] = get[key]
-    print query
+    query['q'] = models.Movie.objects.find(request)
+    if 'r' in get:
+        r = get['r'].split(':')
+        if len(r) == 1: r.append(0)
+        if r[0] == '': r[0] = 0
+        if r[1] == '': r[0] = -1
+        query['i'] = int(r[0])
+        query['o'] = int(r[1])
     return query
 
 def find(request):
-    query = parse_query(request.GET)
+    query = parse_query(request)
     response = {}
-    if "p" in query:
+    if 'k' in query:
         response['movies'] = []
-
+        qs = order_query(query['q'], query['s'])
-        qs = models.Movie.objects.find(**query)
+        qs = qs[query['i']:query['o']]
         p = Paginator(qs, 100)
         for i in p.page_range:
             page = p.page(i)
             for m in page.object_list:
-                  response['movies'].append(m.json(query['p']))
+                  response['movies'].append(m.json(query['k']))
-    elif "g" in query:
+    elif 'g' in query:
+        #FIXME: also filter lists here
         response['groups'] = []
-        name = "name"
+        name = 'name'
-        movies = "movies"
+        movies = 'movies'
-        if query["g"] == "country":
+        movie_qs = query['q']
-            qs = models.Country.objects.values("name").annotate(movies=Count('movies'))
+        _objects = {
-        if query["g"] == "genre":
+            'country': models.Country.objects,
-            qs = models.Genre.objects.values("name").annotate(movies=Count('movies'))
+            'genre': models.Genre.objects,
-        if query["g"] == "language":
+            'language': models.Language.objects,
-            qs = models.Language.objects.values("name").annotate(movies=Count('movies'))
+            'director': models.Person.objects.filter(cast__role='directors'),
-        if query["g"] == "director":
+        }
-            qs = models.Person.objects.filter(cast__role="directors").values("name").annotate(movies=Count('movies'))
+        if query['g'] in _objects:
-        if query["g"] == "year":
+            qs = _objects[query['g']].filter(movies__id__in=movie_qs).values('name').annotate(movies=Count('movies'))
-            qs = models.Movie.objects.values('year').annotate(movies=Count('id'))
+        elif query['g'] == "year":
-            name="year"
+            qs = movie_qs.values('year').annotate(movies=Count('id'))
-        qs = qs[query['o']:query['n']]
+            name='year'
+        qs = order_query(qs, query['s'], '')
+        qs = qs[query['i']:query['o']]
         for i in qs:
-            group = {"name": i[name], "movies": i[movies]}
+            group = {'name': i[name], 'movies': i[movies]}
             response['groups'].append(group)
     else:
-        response['movies'] = models.Movie.objects.all().count()
+        #FIXME: also filter lists here
-        response['files'] = models.MovieFile.objects.all().count()
+        movies = models.Movie.objects.all()
-        r = models.MovieFile.objects.all().aggregate(Count('size'), Count('pixels'), Count('duration'))
+        files = models.MovieFile.objects.all()
+        response['movies'] = movies.count()
+        response['files'] = files.count()
+        r = files.aggregate(Count('size'), Count('pixels'), Count('duration'))
         response['pixels'] = r['pixels__count']
         response['size'] = r['size__count']
         response['duration'] = r['duration__count']
@@ -144,6 +219,9 @@ GET info?oshash=a41cde31c581e11d
 '''
 def file_info(request):
     oshash = request.GET['oshash']
+    f = models.MovieFile.objects.get(oshash=oshash)
+    response = f.json()
+    return render_to_json_response(response)
 
 
 '''
@@ -159,9 +237,20 @@ srt =
 def subtitles(request):
     oshash = request.GET['oshash']
     language = request.GET.get('language', None)
+    if requeset.method == 'POST':
+        user = request.user
+        sub = models.Subtitles.get_or_create(user, oshash, language)
+        sub.srt = request.POST['srt']
+        sub.save()
+    else:
         if language:
-        return srt
+            q = models.Subtitles.objects.filter(movie_file__oshash=oshash, language=language)
-    return movie.subtitle_languages()
+            if q.count() > 0:
+                return HttpResponse(q[0].srt, content_type='text/x-srt')
+        response = {}
+        l = models.Subtitles.objects.filter(movie_file__oshash=oshash).values('language')
+        response['languages'] = [f['language'] for f in l]
+        return render_to_json_response(response)
 
 '''
 GET list
@@ -171,9 +260,27 @@ GET list
       }
     }
 '''
+@login_required
 def list_files(request):
-    files = {}
+    response['files'] = {}
-    return dict(files=files)
+    qs = models.UserFile.filter(user=request.user)
+    p = Paginator(qs, 1000)
+    for i in p.page_range:
+        page = p.page(i)
+        for f in page.object_list:
+              response['files'][f.movie_file.oshash] = {'path': f.path, 'size': f.movie_file.size}
+    return render_to_json_response(response)
+
+def find_files(request):
+    query = parse_query(request)
+    response['files'] = {}
+    qs = models.UserFile.filter(user=request.user).filter(movie_file__movie__id__in=quert['q'])
+    p = Paginator(qs, 1000)
+    for i in p.page_range:
+        page = p.page(i)
+        for f in page.object_list:
+              response['files'][f.movie_file.oshash] = {'path': f.path, 'size': f.movie_file.size}
+    return render_to_json_response(response)
 
 '''
 POST add
@@ -196,13 +303,36 @@ POST add
         "md5":..
       }
 '''
-def add_file(request):
+@login_required
+def add_file(request, archive):
     oshash = request.POST['oshash']
+    archive = models.Archive.objects.get(name=archive)
+    if archive.users.filter(user=request.user).count() == 1:
+        user_file = models.ArchiveFiles.get_or_create(request.user, oshash)
+        user_file.update(request.POST)
+        response = {'status': 200}
+    else:
+        response = {'status': 404}
+    return render_to_json_response(response)
 
 '''
 POST remove?oshash=
 '''
-def remove_file(request):
+@login_required
+def remove_file(request, archive):
     oshash = request.POST['oshash']
+    archive = models.Archive.objects.get(name=archive)
+    models.UserFiles.objects.filter(movie_file__oshash=oshash, user=request.user).delete()
+    response = {'status': 200}
+    return render_to_json_response(response)
+
+'''
+POST preferences/get?key=
+POST preferences/set?key=&value
+'''
+@login_required
+def preferences(request):
+    oshash = request.POST['oshash']
+    return ''
 
 

settings.py

@@ -1,4 +1,10 @@
-# Django settings for oxdata project.
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+# Django settings for oxdb project.
+import os
+from os.path import join
+
+PROJECT_PATH = os.path.normpath(os.path.dirname(__file__))
 
 DEBUG = True
 TEMPLATE_DEBUG = DEBUG
@@ -21,7 +27,7 @@ DATABASE_PORT = ''             # Set to empty string for default. Not used with
 # although not all choices may be available on all operating systems.
 # If running in a Windows environment this must be set to the same as your
 # system time zone.
-TIME_ZONE = 'America/Chicago'
+TIME_ZONE = 'Europe/Berlin'
 
 # Language code for this installation. All choices can be found here:
 # http://www.i18nguy.com/unicode/language-identifiers.html
@@ -35,7 +41,8 @@ USE_I18N = True
 
 # Absolute path to the directory that holds media.
 # Example: "/home/media/media.lawrence.com/"
-MEDIA_ROOT = ''
+MEDIA_ROOT = join(PROJECT_PATH, 'media')
+STATIC_ROOT = join(PROJECT_PATH, 'static')
 
 # URL that handles the media served from MEDIA_ROOT. Make sure to use a
 # trailing slash if there is a path component (optional in other cases).
@@ -45,7 +52,7 @@ MEDIA_URL = ''
 # URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a
 # trailing slash.
 # Examples: "http://foo.com/media/", "/media/".
-ADMIN_MEDIA_PREFIX = '/media/'
+ADMIN_MEDIA_PREFIX = '/admin/media/'
 
 # Make this unique, and don't share it with anybody.
 SECRET_KEY = '3fh^twg4!7*xcise#3d5%ty+^-#9+*f0innkjcco+y0dag_nr-'
@@ -66,9 +73,7 @@ MIDDLEWARE_CLASSES = (
 ROOT_URLCONF = 'oxdb.urls'
 
 TEMPLATE_DIRS = (
-    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".
+    join(PROJECT_PATH, 'templates'),
-    # Always use forward slashes, even on Windows.
-    # Don't forget to use absolute paths, not relative paths.
 )
 
 INSTALLED_APPS = (
@@ -76,5 +81,20 @@ INSTALLED_APPS = (
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.sites',
+    'django.contrib.admin',
+    'django.contrib.humanize',
+
     'oxdb.backend',
 )
+
+try:
+    import socket
+    # hostname = socket.gethostname().replace('.','_')
+    # exec "from host_settings.%s import *" % hostname
+    local_settings_module = socket.gethostname().split(".")[0]
+    if local_settings_module:
+        execfile(os.path.join(PROJECT_PATH, "host_settings", "%s.py" % local_settings_module))
+except ImportError, e:
+    raise e
+
+

urls.py

@@ -1,8 +1,11 @@
 from django.conf.urls.defaults import *
 
+from django.conf import settings
+
 # Uncomment the next two lines to enable the admin:
-# from django.contrib import admin
+from django.contrib import admin
-# admin.autodiscover()
+admin.autodiscover()
+
 
 urlpatterns = patterns('',
     # Example:
@@ -13,5 +16,15 @@ urlpatterns = patterns('',
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
 
     # Uncomment the next line to enable the admin:
-    # (r'^admin/(.*)', admin.site.root),
+    (r'^admin/(.*)', admin.site.root),
 )
+
+if settings.DEBUG:
+    urlpatterns += patterns('',
+        (r'^media/(?P<path>.*)$', 'django.views.static.serve',
+            {'document_root': settings.MEDIA_ROOT}),
+        (r'^static/(?P<path>.*)$', 'django.views.static.serve',
+            {'document_root': settings.STATIC_ROOT}),
+    )
+
+