use ox/ox.web/ox.django

This commit is contained in:
j 2009-10-11 15:00:35 +02:00
parent e38385fcd8
commit f7a9ce1364
10 changed files with 40 additions and 160 deletions

View file

@ -3,13 +3,13 @@
import os import os
from django.conf import settings from django.conf import settings
from oxlib.cache import getUrlUnicode from ox.cache import getUrlUnicode
from oxlib import findRe from ox import findRe
import oxlib.net import ox.net
import oxweb.criterion import ox.web.criterion
import oxweb.karagarga import ox.web.karagarga
import oxweb.imdb import ox.web.imdb
import oxweb.impawards import ox.web.impawards
import models import models
from oxdata.poster.models import PosterCache from oxdata.poster.models import PosterCache
@ -20,10 +20,10 @@ def addPoster(m, url, site, site_id):
p.save() p.save()
def getIds(): def getIds():
for id in oxweb.impawards.getIds(): for id in ox.web.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0: if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
print 'impawards', id print 'impawards', id
data = oxweb.impawards.getData(id) data = ox.web.impawards.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id: if not m.impawards_id:
@ -32,10 +32,10 @@ def getIds():
for poster in data['posters']: for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id) addPoster(m, poster, 'impawards.com', m.imdb_id)
for id in oxweb.criterion.getIds(): for id in ox.web.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0: if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id print 'criterion', id
data = oxweb.criterion.getData(id) data = ox.web.criterion.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id: if not m.criterion_id:
@ -47,10 +47,10 @@ def getIds():
#kg #kg
lastId = models.Karagarga.maxId() lastId = models.Karagarga.maxId()
for id in oxweb.karagarga.getIds(lastId): for id in ox.web.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0: if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
print 'kg', id print 'kg', id
data = oxweb.karagarga.getData(id) data = ox.web.karagarga.getData(id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId']) m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga() kg = models.Karagarga()

View file

@ -8,9 +8,9 @@ from django.db.models import Q, Max
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.utils import simplejson from django.utils import simplejson
import oxweb.wikipedia import ox.web.wikipedia
import oxweb.imdb import ox.web.imdb
from oxlib import stripTags from ox import stripTags
def getMovieIdByImdbId(imdb_id): def getMovieIdByImdbId(imdb_id):
@ -56,8 +56,8 @@ class MovieId(models.Model):
def updateFromWikipedia(self): def updateFromWikipedia(self):
if self.wikipedia_id: if self.wikipedia_id:
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id) wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
data = oxweb.wikipedia.getMovieData(wikipedia_url) data = ox.web.wikipedia.getMovieData(wikipedia_url)
_key = {} _key = {}
for key in ('imdb_id', 'amg_id'): for key in ('imdb_id', 'amg_id'):
if key in data: if key in data:
@ -67,7 +67,7 @@ class MovieId(models.Model):
def updateFromImdb(self): def updateFromImdb(self):
if self.imdb_id: if self.imdb_id:
data = oxweb.imdb.getMovieInfo(self.imdb_id) data = ox.web.imdb.getMovieInfo(self.imdb_id)
_key = { _key = {
'episode title': 'episode_title', 'episode title': 'episode_title',
'series title': 'series_title', 'series title': 'series_title',
@ -76,12 +76,12 @@ class MovieId(models.Model):
if key in data and data[key]: if key in data and data[key]:
setattr(self, _key.get(key, key), data[key]) setattr(self, _key.get(key, key), data[key])
directors = [] directors = []
credits = oxweb.imdb.getMovieCredits(self.imdb_id) credits = ox.web.imdb.getMovieCredits(self.imdb_id)
if 'directors' in credits: if 'directors' in credits:
directors = credits['directors'] directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors]) self.director = ', '.join([stripTags(d[0]) for d in directors])
if not self.wikipedia_id: if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id)) self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
if not self.oxdb_id: if not self.oxdb_id:
self.gen_oxdb_id() self.gen_oxdb_id()
self.save() self.save()

View file

@ -5,7 +5,7 @@ from django.db import models
from django.db.models import Q from django.db.models import Q
from django.contrib.auth.models import User from django.contrib.auth.models import User
from oxdata.utils.shortcuts import render_to_json_response from ox.django.shortcuts import render_to_json_response
def ids(request): def ids(request):
json = {} json = {}

View file

@ -8,11 +8,11 @@ from django.db.models import Q
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
import oxweb.criterion import ox.web.criterion
import oxweb.movieposterdb import ox.web.movieposterdb
import oxweb.karagarga import ox.web.karagarga
import oxweb.imdb import ox.web.imdb
import oxweb.impawards import ox.web.impawards
from oxdata.lookup.models import MovieId, Karagarga from oxdata.lookup.models import MovieId, Karagarga
@ -56,9 +56,9 @@ class PosterCache(models.Model):
def get(self): def get(self):
if not self.image and not self.failed: if not self.image and not self.failed:
try: try:
import oxlib.net import ox.net
name = hashlib.sha1(self.url).hexdigest() name = hashlib.sha1(self.url).hexdigest()
data = oxlib.net.getUrl(self.url) data = ox.net.getUrl(self.url)
self.image.save(name, ContentFile(data)) self.image.save(name, ContentFile(data))
except: except:
self.failed = True self.failed = True
@ -90,28 +90,28 @@ def getPosterUrls(m):
p.save() p.save()
if m.imdb_id: if m.imdb_id:
poster = oxweb.imdb.getMoviePoster(m.imdb_id) poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster: if poster:
addPoster(poster, 'imdb.com', m.imdb_id) addPoster(poster, 'imdb.com', m.imdb_id)
for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']: for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id) addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id: if m.criterion_id:
print 'criterion', m.criterion_id print 'criterion', m.criterion_id
for poster in oxweb.criterion.getData(m.criterion_id)['posters']: for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id) addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id: if m.wikipedia_id:
poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id) poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster: if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0: if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id) addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id: if m.impawards_id:
data = oxweb.impawards.getData(m.impawards_id) data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data: if data and 'imdbId' in data:
for poster in data['posters']: for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id) addPoster(poster, 'impawards.com', m.imdb_id)
for kg in Karagarga.objects.all().filter(movie_id=m): for kg in Karagarga.objects.all().filter(movie_id=m):
data = oxweb.karagarga.getData(kg.karagarga_id) data = ox.web.karagarga.getData(kg.karagarga_id)
if data: if data:
for poster in data['posters']: for poster in data['posters']:
addPoster(poster, 'karagarga.net', kg.karagarga_id) addPoster(poster, 'karagarga.net', kg.karagarga_id)

View file

@ -7,7 +7,7 @@ from django.contrib.auth.models import User
from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404 from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
from django.template import RequestContext from django.template import RequestContext
from oxdata.utils.shortcuts import render_to_json_response from ox.django.shortcuts import render_to_json_response
from oxdata.lookup.models import MovieId from oxdata.lookup.models import MovieId
import models import models

View file

@ -1,113 +0,0 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import sys
import time
import signal
import threading
import atexit
import Queue
_interval = 1.0
_times = {}
_files = []
_running = False
_queue = Queue.Queue()
_lock = threading.Lock()
def _restart(path):
_queue.put(True)
prefix = 'monitor (pid=%d):' % os.getpid()
print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path)
print >> sys.stderr, '%s Triggering process restart.' % prefix
os.kill(os.getpid(), signal.SIGINT)
def _modified(path):
try:
# If path doesn't denote a file and were previously
# tracking it, then it has been removed or the file type
# has changed so force a restart. If not previously
# tracking the file then we can ignore it as probably
# pseudo reference such as when file extracted from a
# collection of modules contained in a zip file.
if not os.path.isfile(path):
return path in _times
# Check for when file last modified.
mtime = os.stat(path).st_mtime
if path not in _times:
_times[path] = mtime
# Force restart when modification time has changed, even
# if time now older, as that could indicate older file
# has been restored.
if mtime != _times[path]:
return True
except:
# If any exception occured, likely that file has been
# been removed just before stat(), so force a restart.
return True
return False
def _monitor():
while 1:
# Check modification times on all files in sys.modules.
for module in sys.modules.values():
if not hasattr(module, '__file__'):
continue
path = getattr(module, '__file__')
if not path:
continue
if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']:
path = path[:-1]
if _modified(path):
return _restart(path)
# Check modification times on files which have
# specifically been registered for monitoring.
for path in _files:
if _modified(path):
return _restart(path)
# Go to sleep for specified interval.
try:
return _queue.get(timeout=_interval)
except:
pass
_thread = threading.Thread(target=_monitor)
_thread.setDaemon(True)
def _exiting():
try:
_queue.put(True)
except:
pass
_thread.join()
atexit.register(_exiting)
def track(path):
if not path in _files:
_files.append(path)
def start(interval=1.0):
global _interval
if interval < _interval:
_interval = interval
global _running
_lock.acquire()
if not _running:
_running = True
_thread.start()
_lock.release()

View file

@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.http import HttpResponse
from django.utils import simplejson
def render_to_json_response(dictionary, content_type="text/json"):
return HttpResponse(simplejson.dumps(dictionary), content_type=content_type)

View file

@ -1,4 +1,6 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django -e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
South==0.6.1 South==0.6.1
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxlib -e bzr+http://code.0xdb.org/python-oxlib/#egg=python-ox
-e bzr+http://code.0xdb.org/python-oxweb/#egg=python-oxweb -e bzr+http://code.0xdb.org/python-oxweb/#egg=python-oxweb
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxdjango

View file

@ -15,7 +15,7 @@ sys.path.append(root_dir)
sys.path.append(os.path.join(root_dir, project_module)) sys.path.append(os.path.join(root_dir, project_module))
#reload if this django.wsgi gets touched #reload if this django.wsgi gets touched
from utils import monitor from ox.django import monitor
monitor.start(interval=1.0) monitor.start(interval=1.0)
monitor.track(os.path.abspath(os.path.dirname(__file__))) monitor.track(os.path.abspath(os.path.dirname(__file__)))