move files for new virtualenv setup

oxdata/lookup/admin.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+
+import models
+from django.contrib import admin
+
+class MovieIdAdmin(admin.ModelAdmin):
+    search_fields = ['title', 'imdb_id']
+admin.site.register(models.MovieId, MovieIdAdmin)
+

oxdata/lookup/cache.py

@@ -0,0 +1,62 @@
+# -*- coding: UTF-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import os
+
+from django.conf import settings
+from oxlib.cache import getUrlUnicode
+from oxlib import findRe
+import oxlib.net
+import oxweb.criterion
+import oxweb.karagarga
+import oxweb.imdb
+import oxweb.impawards
+
+import models
+from oxdata.poster.models import PosterCache
+
+def addPoster(m, url, site, site_id):
+    if PosterCache.objects.all().filter(url=url).count() == 0:
+        p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
+        p.save()
+
+def getIds():
+    for id in oxweb.impawards.getIds():
+        if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
+            print 'impawards', id
+            data = oxweb.impawards.getData(id)
+            if data and 'imdbId' in data:
+                m = models.getMovieIdByImdbId(data['imdbId'])
+                if not m.impawards_id:
+                    m.impawards_id = id
+                    m.save()
+                for poster in data['posters']:
+                        addPoster(m, poster, 'impawards.com', m.imdb_id)
+
+    for id in oxweb.criterion.getIds():
+        if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
+            print 'criterion', id
+            data = oxweb.criterion.getData(id)
+            if data and 'imdbId' in data:
+                m = models.getMovieIdByImdbId(data['imdbId'])
+                if not m.criterion_id:
+                    m.criterion_id = id
+                    m.save()
+                addPoster(m, poster, 'criterion.com', m.criterion_id)
+            else:
+                print data['title'], "no imdbId"
+
+    #kg
+    lastId = models.Karagarga.maxId()
+    for id in oxweb.karagarga.getIds(lastId):
+        if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
+            print 'kg', id
+            data = oxweb.karagarga.getData(id)
+            if data and 'imdbId' in data:
+                m = models.getMovieIdByImdbId(data['imdbId'])
+                kg = models.Karagarga()
+                kg.movie_id = m
+                kg.karagarga_id = id
+                kg.save()
+                for poster in data['posters']:
+                    addPoster(m, poster, 'karagarga.net', kg.karagarga_id)
+

oxdata/lookup/management/commands/cache_ids.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+
+import lookup.cache
+
+class Command(BaseCommand):
+    """
+    load new ids into cache
+    """
+    help = 'load ids from sites that dont support search.'
+    args = ''
+
+    def handle(self, **options):
+        import oxdata.lookup.cache
+        lookup.cache.getIds()
+

oxdata/lookup/migrations/0001_initial.py

@@ -0,0 +1,78 @@
+# -*- coding: utf-8 -*-
+
+from south.db import db
+from django.db import models
+from oxdata.lookup.models import *
+
+class Migration:
+    
+    def forwards(self, orm):
+        
+        # Adding model 'MovieId'
+        db.create_table('lookup_movieid', (
+            ('id', orm['lookup.MovieId:id']),
+            ('created', orm['lookup.MovieId:created']),
+            ('modified', orm['lookup.MovieId:modified']),
+            ('title', orm['lookup.MovieId:title']),
+            ('year', orm['lookup.MovieId:year']),
+            ('director', orm['lookup.MovieId:director']),
+            ('series_title', orm['lookup.MovieId:series_title']),
+            ('episode_title', orm['lookup.MovieId:episode_title']),
+            ('season', orm['lookup.MovieId:season']),
+            ('episode', orm['lookup.MovieId:episode']),
+            ('oxdb_id', orm['lookup.MovieId:oxdb_id']),
+            ('imdb_id', orm['lookup.MovieId:imdb_id']),
+            ('amg_id', orm['lookup.MovieId:amg_id']),
+            ('wikipedia_id', orm['lookup.MovieId:wikipedia_id']),
+            ('criterion_id', orm['lookup.MovieId:criterion_id']),
+            ('impawards_id', orm['lookup.MovieId:impawards_id']),
+        ))
+        db.send_create_signal('lookup', ['MovieId'])
+        
+        # Adding model 'Karagarga'
+        db.create_table('lookup_karagarga', (
+            ('id', orm['lookup.Karagarga:id']),
+            ('movie_id', orm['lookup.Karagarga:movie_id']),
+            ('karagarga_id', orm['lookup.Karagarga:karagarga_id']),
+        ))
+        db.send_create_signal('lookup', ['Karagarga'])
+        
+    
+    
+    def backwards(self, orm):
+        
+        # Deleting model 'MovieId'
+        db.delete_table('lookup_movieid')
+        
+        # Deleting model 'Karagarga'
+        db.delete_table('lookup_karagarga')
+        
+    
+    
+    models = {
+        'lookup.karagarga': {
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'karagarga_id': ('django.db.models.fields.IntegerField', [], {'unique': 'True'}),
+            'movie_id': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'karagarga_ids'", 'to': "orm['lookup.MovieId']"})
+        },
+        'lookup.movieid': {
+            'amg_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'criterion_id': ('django.db.models.fields.IntegerField', [], {'default': 'None', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'director': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'episode': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'episode_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'imdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '7', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'impawards_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'oxdb_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '42', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'season': ('django.db.models.fields.IntegerField', [], {'default': '-1'}),
+            'series_title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1000', 'blank': 'True'}),
+            'wikipedia_id': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '255', 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'year': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '4', 'blank': 'True'})
+        }
+    }
+    
+    complete_apps = ['lookup']

oxdata/lookup/models.py

@@ -0,0 +1,128 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import os
+import hashlib
+
+from django.db import models
+from django.db.models import Q, Max
+from django.contrib.auth.models import User
+from django.utils import simplejson
+
+import oxweb.wikipedia
+import oxweb.imdb
+from oxlib import stripTags
+
+
+def getMovieIdByImdbId(imdb_id):
+    try:
+        m = MovieId.objects.get(imdb_id=imdb_id)
+    except MovieId.DoesNotExist:
+        m = MovieId()
+        m.imdb_id = imdb_id
+        m.save()
+        #m.updateFromImdb()
+    return m
+
+class MovieId(models.Model):
+    created = models.DateTimeField(auto_now_add=True)
+    modified = models.DateTimeField(auto_now=True)
+
+    title = models.CharField(max_length=1000, blank=True, default='')
+    year = models.CharField(max_length=4, blank=True, default='')
+    director = models.CharField(max_length=1000, blank=True, default='')
+    series_title = models.CharField(max_length=1000, blank=True, default='')
+    episode_title = models.CharField(max_length=1000, blank=True, default='')
+    season = models.IntegerField(default=-1)
+    episode = models.IntegerField(default=-1)
+
+    oxdb_id = models.CharField(max_length=42, unique=True, blank=True, null=True,  default=None)
+    imdb_id = models.CharField(max_length=7, unique=True, blank=True, null=True,  default=None)
+
+    amg_id = models.IntegerField(unique=True, blank=True, null=True,  default=None)
+    wikipedia_id = models.CharField(unique=True, max_length=255, blank=True, null=True,  default=None)
+    criterion_id = models.IntegerField(unique=True, blank=True, null=True,  default=None)
+    impawards_id = models.CharField(max_length=255, unique=True, blank=True, null=True,  default=None)
+
+    #FIXME: look into other ids
+    #what about tv.com ids/urls for tv episodes
+    
+    #movieposterdb can be mapped via imdbid and does not need to be saved here
+
+    def __unicode__(self):
+        id = self.imdb_id
+        if not id:
+            id = self.id
+        return '%s (%s)' % (self.title, id)
+
+    def updateFromWikipedia(self):
+        if self.wikipedia_id:
+            wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id)
+            data = oxweb.wikipedia.getMovieData(wikipedia_url)
+            _key = {}
+            for key in ('imdb_id', 'amg_id'):
+                if key in data:
+                    if data[key] and not getattr(self, _key.get(key, key)):
+                        setattr(self, _key.get(key, key), data[key])
+            self.save()
+
+    def updateFromImdb(self):
+        if self.imdb_id:
+            data = oxweb.imdb.getMovieInfo(self.imdb_id)
+            _key = {
+                'episode title': 'episode_title',
+                'series title': 'series_title',
+            }
+            for key in ('title', 'year', 'series title', 'episode title', 'season', 'episode'):
+                if key in data and data[key]:
+                    setattr(self, _key.get(key, key), data[key])
+            directors = []
+            credits = oxweb.imdb.getMovieCredits(self.imdb_id)
+            if 'directors' in credits:
+                directors = credits['directors']
+            self.director = ', '.join([stripTags(d[0]) for d in directors])
+            if not self.wikipedia_id:
+                self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
+            if not self.oxdb_id:
+                self.gen_oxdb_id()
+            self.save()
+
+    def gen_oxdb_id(self):
+        oxid_value = u"\n".join([self.title, self.director, self.year])
+        oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()
+        if self.episode > -1:
+            oxid_value = u"\n".join([self.series_title, "%02d" % self.season])
+            oxid = hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
+            oxid_value = u"\n".join(["%02d" % self.episode, self.episode_title, self.director, self.year])
+            oxid += hashlib.sha1(oxid_value.encode('utf-8')).hexdigest()[:20]
+        self.oxdb_id = u"0x" + oxid
+        self.save()
+
+    def json(self):
+        json = {}
+        keys = [
+            'title',
+            'director',
+            'year',
+            'imdb_id',
+            'oxdb_id',
+            'amg_id',
+            'wikipedia_id',
+            'criterion_id',
+            'impawards_id',
+        ]
+        if self.episode > -1:
+            keys += ['episode', 'season', 'episode_title', 'series_title']
+        for key in keys:
+            value = getattr(self, key)
+            if value:
+                json[key] = value
+        return simplejson.dumps(json, indent=4)
+
+class Karagarga(models.Model):
+    movie_id = models.ForeignKey(MovieId, related_name='karagarga_ids', default=None)
+    karagarga_id = models.IntegerField(unique=True)
+
+    @classmethod
+    def maxId(cls):
+        return cls.objects.aggregate(Max('karagarga_id'))['karagarga_id__max']
+    

oxdata/lookup/urls.py

@@ -0,0 +1,7 @@
+from django.conf.urls.defaults import *
+
+urlpatterns = patterns('oxdata.lookup.views',
+    (r'^$', 'ids'),
+    (r'^urls$', 'urls'),
+)
+

oxdata/lookup/views.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=4:sts=4:ts=4
+import os.path
+from django.db import models
+from django.db.models import Q
+from django.contrib.auth.models import User
+
+from oxdata.utils.shortcuts import render_to_json_response
+
+def ids(request):
+    json = {}
+    return render_to_json_response(json)
+
+def urls(request):
+    json = {}
+    return render_to_json_response(json)
+