posters all over

This commit is contained in:
j 2009-07-15 18:39:28 +02:00
parent bc59f543cb
commit 0243766f8f
25 changed files with 211 additions and 211 deletions

View file

@ -1 +1,3 @@
data data
cache
media

View file

View file

@ -1,49 +0,0 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.criterion
from oxdata.lookup.models import MovieId, getMovieIdByImdbId
def getPoster(id, url=None):
dirname = os.path.join(settings.DATA_ROOT, 'criterion.com', id)
filename = os.path.join(dirname, 'poster.jpg')
filename = os.path.normpath(filename)
if not os.path.exists(filename):
if not url:
data = oxweb.criterion.getData(id)
url = data['posterUrl']
oxlib.net.saveUrl(url, filename)
return filename
def archivePosters(init=False):
for criterionId in oxweb.criterion.getIds():
data = oxweb.criterion.getData(criterionId)
imdbId = data['imdbId']
try:
m = MovieId.objects.get(criterion_id=criterionId)
if imdbId and not m.imdb_id:
m.imdb_id = imdbId
m.save()
except MovieId.DoesNotExist:
if imdbId:
m = getMovieIdByImdbId(imdbId)
else:
m = MovieId()
m.criterion_id = criterionId
m.save()
url = data['posterUrl']
getPoster(criterionId, url)
def cron():
archivePosters()
def init():
archivePosters(True)

View file

@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

View file

@ -1 +0,0 @@
# Create your views here.

View file

View file

@ -1,47 +0,0 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.impawards
def getPosterFilename(id, url):
dirname = os.path.join(settings.DATA_ROOT, 'impawards.com', id[:1], id[:4], id)
filename = os.path.join(dirname, os.path.split(url)[1])
filename = os.path.normpath(filename)
return filename
def getPoster(id, url):
filename = getPosterFilename(id, url)
oxlib.net.saveUrl(url, filename)
return filename
def archivePosters(init=False):
html = getUrlUnicode('http://impawards.com/archives/latest.html', timeout = 0)
pages = int(findRe(html, '<a href = page(.*?).html>'))
for page in range(pages + 1, 0, -1):
if page <= pages:
html = getUrlUnicode('http://impawards.com/archives/page%s.html' % page, timeout = -1)
urls = oxweb.impawards.parseArchivePage(html)
for url in urls:
html = getUrlUnicode(url, timeout = -1)
data = oxweb.impawards.parseMoviePage(html)
service = 'impawards'
url = data['posterUrl']
imdbId = data['imdbId']
filename = getPosterFilename(imdbId, url)
if not os.path.exists(filename):
getPoster(imdbId, url)
elif not init:
return
def cron():
archivePosters()
def init():
archivePosters(True)

View file

@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

View file

@ -1 +0,0 @@
# Create your views here.

View file

View file

@ -1,33 +0,0 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import hashlib
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.movieposterdb
def getPosterFilename(id, url):
id = str(id)
dirname = os.path.join(settings.DATA_ROOT, 'karagarga.net', id[:1], id)
url_hash = hashlib.sha1(url).hexdigest()
filename = os.path.join(dirname, '%s.jpg' % url_hash)
filename = os.path.normpath(filename)
return filename
def getPoster(id, url):
filename = getPosterFilename(id, url)
oxlib.net.saveUrl(url, filename)
return filename
def archivePosters(init=False):
return
def cron():
archivePosters()
def init():
archivePosters(True)

View file

@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

View file

@ -1 +0,0 @@
# Create your views here.

59
lookup/cache.py Normal file
View file

@ -0,0 +1,59 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.criterion
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
import models
from oxdata.poster.models import PosterCache
def addPoster(m, url, site, site_id):
if PosterCache.objects.all().filter(url=url).count() == 0:
p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
p.save()
def getIds():
for id in oxweb.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
data = oxweb.impawards.getData(id)
if data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id:
m.impawards_id = id
m.save()
for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id)
for criterionId in oxweb.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
data = oxweb.criterion.getData(criterionId)
if data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id:
m.criterion_id = id
m.save()
addPoster(m, poster, 'criterion.com', m.criterion_id)
#kg
lastId = models.Karagarga.maxId()
for karagargaId in oxweb.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=karagargaId).count() == 0:
data = oxweb.karagarga.getData(karagargaId)
if data:
if 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga()
kg.movie_id = m
kg.karagarga_id = karagargaId
kg.save()
#fixme, what to do else?
for poster in data['posters']:
addPoster(poster, 'karagarga.net', kg.karagarga_id)

View file

@ -4,10 +4,10 @@ import os
import hashlib import hashlib
from django.db import models from django.db import models
from django.db.models import Q from django.db.models import Q, Max
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.utils import simplejson
import simplejson
import oxweb.wikipedia import oxweb.wikipedia
import oxweb.imdb import oxweb.imdb
from oxlib import stripTags from oxlib import stripTags
@ -20,6 +20,7 @@ def getMovieIdByImdbId(imdb_id):
m = MovieId() m = MovieId()
m.imdb_id = imdb_id m.imdb_id = imdb_id
m.save() m.save()
#m.updateFromImdb()
return m return m
class MovieId(models.Model): class MovieId(models.Model):
@ -79,6 +80,8 @@ class MovieId(models.Model):
if 'directors' in credits: if 'directors' in credits:
directors = credits['directors'] directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors]) self.director = ', '.join([stripTags(d[0]) for d in directors])
if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
if not self.oxdb_id: if not self.oxdb_id:
self.gen_oxdb_id() self.gen_oxdb_id()
self.save() self.save()
@ -118,3 +121,7 @@ class Karagarga(models.Model):
movie_id = models.ForeignKey(MovieId, related_name='karagarga_ids', default=None) movie_id = models.ForeignKey(MovieId, related_name='karagarga_ids', default=None)
karagarga_id = models.IntegerField(unique=True) karagarga_id = models.IntegerField(unique=True)
@classmethod
def maxId(cls):
return cls.objects.aggregate(Max('karagarga_id'))['karagarga_id__max']

View file

@ -1,34 +0,0 @@
# -*- coding: UTF-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import hashlib
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.movieposterdb
def getPoster(id, url=''):
if not url:
urls = oxweb.movieposterdb.getPosterUrls(id)
if urls:
url = urls[0]
else:
return
dirname = os.path.join(settings.DATA_ROOT, 'movieposterdb.com', id[:1], id[:4], id)
url_hash = hashlib.sha1(url).hexdigest()
filename = os.path.join(dirname, '%s.jpg' % url_hash)
filename = os.path.normpath(filename)
oxlib.net.saveUrl(url, filename)
return filename
def archivePosters(init=False):
return
def cron():
archivePosters()
def init():
archivePosters(True)

View file

@ -1,3 +0,0 @@
from django.db import models
# Create your models here.

View file

@ -1 +0,0 @@
# Create your views here.

View file

@ -15,17 +15,31 @@ import oxweb.imdb
from oxdata.lookup.models import MovieId, Karagarga from oxdata.lookup.models import MovieId, Karagarga
def getPosters(movie_id):
class Poster(models.Model): getPosterUrls(movie_id)
created = models.DateTimeField(auto_now_add=True) posters = {}
modified = models.DateTimeField(auto_now=True) for p in PosterCache.objects.all().filter(movie_id=movie_id):
if p.site not in posters:
movie_id = models.ForeignKey(MovieId, related_name='poster') posters[p.site] = []
url = models.CharField(max_length=255) poster = p.get()
if poster:
pjson = {}
pjson['url'] = poster._get_url()
pjson['width'] = poster.width
pjson['height'] = poster.height
posters[p.site].append(pjson)
for p in posters:
if not posters[p]:
del posters[p]
return posters
def poster_path(instance, filename): def poster_path(instance, filename):
id = instance.service_id url_hash = hashlib.sha1(instance.url).hexdigest()
return os.path.join('poster', instance.service, id[:1], id[:4], id, filename) ext = 'jpg'
if filename.endswith('.png'):
ext = 'png'
name = "%s.%s" % (url_hash, ext)
return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
class PosterCache(models.Model): class PosterCache(models.Model):
created = models.DateTimeField(auto_now_add=True) created = models.DateTimeField(auto_now_add=True)
@ -33,26 +47,50 @@ class PosterCache(models.Model):
movie_id = models.ForeignKey(MovieId, related_name='postercache') movie_id = models.ForeignKey(MovieId, related_name='postercache')
url = models.CharField(max_length=1024) url = models.CharField(max_length=1024)
service = models.CharField(max_length=255) site = models.CharField(max_length=255)
service_id = models.CharField(max_length=42) site_id = models.CharField(max_length=42)
image = models.ImageField(max_length=255, upload_to=poster_path) image = models.ImageField(max_length=255, upload_to=poster_path)
failed = models.BooleanField(default=False)
def get(self): def get(self):
if not self.image: if not self.image and not self.failed:
import oxlib.net try:
ext = 'jpg' import oxlib.net
if self.url.endswith('.png'): name = hashlib.sha1(self.url).hexdigest()
ext = 'png' data = oxlib.net.getUrl(self.url)
name = "%s.%s" % (hashlib.sha1(self.url).hexdigest(), ext) self.image.save(name, ContentFile(data))
data = oxlib.net.getUrl(self.url) except:
self.failed = True
self.image.save(name, ContentFile(data)) self.save()
return self.image return self.image
def manual_poster_path(instance, filename):
url_hash = hashlib.sha1('%s' % instance.id).hexdigest()
ext = 'jpg'
if filename.endswith('.png'):
ext = 'png'
name = "%s.%s" % (url_hash, ext)
return os.path.join('posters', url_hash[:2], url_hash[2:4], url_hash[4:6], name)
class Poster(models.Model):
created = models.DateTimeField(auto_now_add=True)
modified = models.DateTimeField(auto_now=True)
movie_id = models.ForeignKey(MovieId, related_name='poster')
poster = models.ForeignKey(PosterCache, default=None, blank=True)
image = models.ImageField(max_length=255, upload_to=manual_poster_path)
def get(self):
if self.image:
return self.image
if self.poster:
return self.poster.get()
#FIXME: select poster
return None
def getPosterUrls(m): def getPosterUrls(m):
def addPoster(url, service, service_id): def addPoster(url, site, site_id):
p = PosterCache(url=url, service=service, service_id=service_id, movie_id=m) p = PosterCache(url=url, site=site, site_id=site_id, movie_id=m)
p.save() p.save()
if m.imdb_id: if m.imdb_id:
@ -75,4 +113,6 @@ def getPosterUrls(m):
for poster in data['posters']: for poster in data['posters']:
if PosterCache.objects.all().filter(url=poster).count() == 0: if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'karagarga.net', kg.karagarga_id) addPoster(poster, 'karagarga.net', kg.karagarga_id)
else:
kg.delete()

View file

@ -2,5 +2,6 @@ from django.conf.urls.defaults import *
urlpatterns = patterns('oxdata.poster.views', urlpatterns = patterns('oxdata.poster.views',
(r'^$', 'poster'), (r'^$', 'poster'),
(r'^json$', 'poster_json'),
) )

View file

@ -4,11 +4,33 @@ import os.path
from django.db import models from django.db import models
from django.db.models import Q from django.db.models import Q
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
from django.template import RequestContext
from oxdata.utils.shortcuts import render_to_json_response from oxdata.utils.shortcuts import render_to_json_response
from oxdata.lookup.models import MovieId
import models
def poster(request): def poster(request):
#FIXME: get MovieId from provided id and redirect to poster url #FIXME: get MovieId from provided id and redirect to poster url
return if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
context = RequestContext(request, {'imdb': imdbId})
return render_to_response('poster.html', context)
def poster_json(request):
#FIXME: get MovieId from provided id and redirect to poster url
if 'imdb' in request.GET:
imdbId = request.GET['imdb']
movie_id = MovieId.objects.get(imdb_id=imdbId)
elif 'oxdb' in request.GET:
oxdbId = request.GET['oxdb']
movie_id = MovieId.objects.get(oxdb_id=oxdbId)
json = models.getPosters(movie_id)
return render_to_json_response(json)

View file

@ -4,9 +4,9 @@
import os import os
from os.path import join from os.path import join
PROJECT_PATH = os.path.dirname(__file__) PROJECT_PATH = os.path.normpath(os.path.dirname(__file__))
DEBUG = False DEBUG = True
TEMPLATE_DEBUG = DEBUG TEMPLATE_DEBUG = DEBUG
ADMINS = ( ADMINS = (
@ -44,6 +44,9 @@ USE_I18N = True
MEDIA_ROOT = join(PROJECT_PATH, 'media') MEDIA_ROOT = join(PROJECT_PATH, 'media')
STATIC_ROOT = join(PROJECT_PATH, 'static') STATIC_ROOT = join(PROJECT_PATH, 'static')
DATA_ROOT = join(PROJECT_PATH, 'data') DATA_ROOT = join(PROJECT_PATH, 'data')
CACHE_ROOT = join(PROJECT_PATH, 'cache')
os.environ['oxCACHE'] = CACHE_ROOT
# URL that handles the media served from MEDIA_ROOT. Make sure to use a # URL that handles the media served from MEDIA_ROOT. Make sure to use a
@ -85,11 +88,6 @@ INSTALLED_APPS = (
'django.contrib.sites', 'django.contrib.sites',
'django.contrib.admin', 'django.contrib.admin',
'django.contrib.humanize', 'django.contrib.humanize',
'oxdata.criterion',
'oxdata.impawards',
'oxdata.karagarga',
'oxdata.impawards',
'oxdata.movieposterdb',
'oxdata.lookup', 'oxdata.lookup',
'oxdata.poster', 'oxdata.poster',

41
templates/poster.html Normal file
View file

@ -0,0 +1,41 @@
<!DOCTYPE html>
<html>
<head>
<style>
.hide {
display: none;
}
</style>
<script src="/static/js/jquery.js"></script>
<script>
$(function(){
$.getJSON('/poster/json?imdb={{ imdb }}', function(posters) {
$("#loading").hide();
$.each(posters, function(key, urls) {
var id = 'row' + key.replace('.', '_');
var row = '<tr id="'+id+'">';
row += '<td id="'+id+'_title">' + key + '</td>';
$.each(urls, function(i, poster) {
row += '<td>';
row += '<img src="'+poster.url+'" height="256">';
row += '<div>' + poster.width + 'x'+ poster.height + '</div>';
row += '</td>';
});
row += '</tr>';
$("#posters").append(row);
$('#'+id+'_title').click(function() {
$('#'+id+' img, #'+id+' div').toggleClass("hide");
});
});
});
});
</script>
</head>
<body>
<div id="loading">loading...</div>
<table id="posters">
</table>
</body>
</html>

View file

@ -13,3 +13,12 @@ urlpatterns = patterns('',
(r'^admin/(.*)', admin.site.root), (r'^admin/(.*)', admin.site.root),
) )
if settings.DEBUG:
urlpatterns += patterns('',
(r'^media/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.MEDIA_ROOT}),
(r'^static/(?P<path>.*)$', 'django.views.static.serve',
{'document_root': settings.STATIC_ROOT}),
)