use ox/ox.web/ox.django
This commit is contained in:
parent
e38385fcd8
commit
f7a9ce1364
10 changed files with 40 additions and 160 deletions
|
@ -3,13 +3,13 @@
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from oxlib.cache import getUrlUnicode
|
from ox.cache import getUrlUnicode
|
||||||
from oxlib import findRe
|
from ox import findRe
|
||||||
import oxlib.net
|
import ox.net
|
||||||
import oxweb.criterion
|
import ox.web.criterion
|
||||||
import oxweb.karagarga
|
import ox.web.karagarga
|
||||||
import oxweb.imdb
|
import ox.web.imdb
|
||||||
import oxweb.impawards
|
import ox.web.impawards
|
||||||
|
|
||||||
import models
|
import models
|
||||||
from oxdata.poster.models import PosterCache
|
from oxdata.poster.models import PosterCache
|
||||||
|
@ -20,10 +20,10 @@ def addPoster(m, url, site, site_id):
|
||||||
p.save()
|
p.save()
|
||||||
|
|
||||||
def getIds():
|
def getIds():
|
||||||
for id in oxweb.impawards.getIds():
|
for id in ox.web.impawards.getIds():
|
||||||
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
|
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
|
||||||
print 'impawards', id
|
print 'impawards', id
|
||||||
data = oxweb.impawards.getData(id)
|
data = ox.web.impawards.getData(id)
|
||||||
if data and 'imdbId' in data:
|
if data and 'imdbId' in data:
|
||||||
m = models.getMovieIdByImdbId(data['imdbId'])
|
m = models.getMovieIdByImdbId(data['imdbId'])
|
||||||
if not m.impawards_id:
|
if not m.impawards_id:
|
||||||
|
@ -32,10 +32,10 @@ def getIds():
|
||||||
for poster in data['posters']:
|
for poster in data['posters']:
|
||||||
addPoster(m, poster, 'impawards.com', m.imdb_id)
|
addPoster(m, poster, 'impawards.com', m.imdb_id)
|
||||||
|
|
||||||
for id in oxweb.criterion.getIds():
|
for id in ox.web.criterion.getIds():
|
||||||
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
|
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
|
||||||
print 'criterion', id
|
print 'criterion', id
|
||||||
data = oxweb.criterion.getData(id)
|
data = ox.web.criterion.getData(id)
|
||||||
if data and 'imdbId' in data:
|
if data and 'imdbId' in data:
|
||||||
m = models.getMovieIdByImdbId(data['imdbId'])
|
m = models.getMovieIdByImdbId(data['imdbId'])
|
||||||
if not m.criterion_id:
|
if not m.criterion_id:
|
||||||
|
@ -47,10 +47,10 @@ def getIds():
|
||||||
|
|
||||||
#kg
|
#kg
|
||||||
lastId = models.Karagarga.maxId()
|
lastId = models.Karagarga.maxId()
|
||||||
for id in oxweb.karagarga.getIds(lastId):
|
for id in ox.web.karagarga.getIds(lastId):
|
||||||
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
|
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
|
||||||
print 'kg', id
|
print 'kg', id
|
||||||
data = oxweb.karagarga.getData(id)
|
data = ox.web.karagarga.getData(id)
|
||||||
if data and 'imdbId' in data:
|
if data and 'imdbId' in data:
|
||||||
m = models.getMovieIdByImdbId(data['imdbId'])
|
m = models.getMovieIdByImdbId(data['imdbId'])
|
||||||
kg = models.Karagarga()
|
kg = models.Karagarga()
|
||||||
|
|
|
@ -8,9 +8,9 @@ from django.db.models import Q, Max
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.utils import simplejson
|
from django.utils import simplejson
|
||||||
|
|
||||||
import oxweb.wikipedia
|
import ox.web.wikipedia
|
||||||
import oxweb.imdb
|
import ox.web.imdb
|
||||||
from oxlib import stripTags
|
from ox import stripTags
|
||||||
|
|
||||||
|
|
||||||
def getMovieIdByImdbId(imdb_id):
|
def getMovieIdByImdbId(imdb_id):
|
||||||
|
@ -56,8 +56,8 @@ class MovieId(models.Model):
|
||||||
|
|
||||||
def updateFromWikipedia(self):
|
def updateFromWikipedia(self):
|
||||||
if self.wikipedia_id:
|
if self.wikipedia_id:
|
||||||
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id)
|
wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
|
||||||
data = oxweb.wikipedia.getMovieData(wikipedia_url)
|
data = ox.web.wikipedia.getMovieData(wikipedia_url)
|
||||||
_key = {}
|
_key = {}
|
||||||
for key in ('imdb_id', 'amg_id'):
|
for key in ('imdb_id', 'amg_id'):
|
||||||
if key in data:
|
if key in data:
|
||||||
|
@ -67,7 +67,7 @@ class MovieId(models.Model):
|
||||||
|
|
||||||
def updateFromImdb(self):
|
def updateFromImdb(self):
|
||||||
if self.imdb_id:
|
if self.imdb_id:
|
||||||
data = oxweb.imdb.getMovieInfo(self.imdb_id)
|
data = ox.web.imdb.getMovieInfo(self.imdb_id)
|
||||||
_key = {
|
_key = {
|
||||||
'episode title': 'episode_title',
|
'episode title': 'episode_title',
|
||||||
'series title': 'series_title',
|
'series title': 'series_title',
|
||||||
|
@ -76,12 +76,12 @@ class MovieId(models.Model):
|
||||||
if key in data and data[key]:
|
if key in data and data[key]:
|
||||||
setattr(self, _key.get(key, key), data[key])
|
setattr(self, _key.get(key, key), data[key])
|
||||||
directors = []
|
directors = []
|
||||||
credits = oxweb.imdb.getMovieCredits(self.imdb_id)
|
credits = ox.web.imdb.getMovieCredits(self.imdb_id)
|
||||||
if 'directors' in credits:
|
if 'directors' in credits:
|
||||||
directors = credits['directors']
|
directors = credits['directors']
|
||||||
self.director = ', '.join([stripTags(d[0]) for d in directors])
|
self.director = ', '.join([stripTags(d[0]) for d in directors])
|
||||||
if not self.wikipedia_id:
|
if not self.wikipedia_id:
|
||||||
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
|
self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
|
||||||
if not self.oxdb_id:
|
if not self.oxdb_id:
|
||||||
self.gen_oxdb_id()
|
self.gen_oxdb_id()
|
||||||
self.save()
|
self.save()
|
||||||
|
|
|
@ -5,7 +5,7 @@ from django.db import models
|
||||||
from django.db.models import Q
|
from django.db.models import Q
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
|
|
||||||
from oxdata.utils.shortcuts import render_to_json_response
|
from ox.django.shortcuts import render_to_json_response
|
||||||
|
|
||||||
def ids(request):
|
def ids(request):
|
||||||
json = {}
|
json = {}
|
||||||
|
|
|
@ -8,11 +8,11 @@ from django.db.models import Q
|
||||||
from django.contrib.auth.models import User
|
from django.contrib.auth.models import User
|
||||||
from django.core.files.base import ContentFile
|
from django.core.files.base import ContentFile
|
||||||
|
|
||||||
import oxweb.criterion
|
import ox.web.criterion
|
||||||
import oxweb.movieposterdb
|
import ox.web.movieposterdb
|
||||||
import oxweb.karagarga
|
import ox.web.karagarga
|
||||||
import oxweb.imdb
|
import ox.web.imdb
|
||||||
import oxweb.impawards
|
import ox.web.impawards
|
||||||
|
|
||||||
from oxdata.lookup.models import MovieId, Karagarga
|
from oxdata.lookup.models import MovieId, Karagarga
|
||||||
|
|
||||||
|
@ -56,9 +56,9 @@ class PosterCache(models.Model):
|
||||||
def get(self):
|
def get(self):
|
||||||
if not self.image and not self.failed:
|
if not self.image and not self.failed:
|
||||||
try:
|
try:
|
||||||
import oxlib.net
|
import ox.net
|
||||||
name = hashlib.sha1(self.url).hexdigest()
|
name = hashlib.sha1(self.url).hexdigest()
|
||||||
data = oxlib.net.getUrl(self.url)
|
data = ox.net.getUrl(self.url)
|
||||||
self.image.save(name, ContentFile(data))
|
self.image.save(name, ContentFile(data))
|
||||||
except:
|
except:
|
||||||
self.failed = True
|
self.failed = True
|
||||||
|
@ -90,28 +90,28 @@ def getPosterUrls(m):
|
||||||
p.save()
|
p.save()
|
||||||
|
|
||||||
if m.imdb_id:
|
if m.imdb_id:
|
||||||
poster = oxweb.imdb.getMoviePoster(m.imdb_id)
|
poster = ox.web.imdb.getMoviePoster(m.imdb_id)
|
||||||
if poster:
|
if poster:
|
||||||
addPoster(poster, 'imdb.com', m.imdb_id)
|
addPoster(poster, 'imdb.com', m.imdb_id)
|
||||||
for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']:
|
for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
|
||||||
addPoster(poster, 'movieposterdb.com', m.imdb_id)
|
addPoster(poster, 'movieposterdb.com', m.imdb_id)
|
||||||
if m.criterion_id:
|
if m.criterion_id:
|
||||||
print 'criterion', m.criterion_id
|
print 'criterion', m.criterion_id
|
||||||
for poster in oxweb.criterion.getData(m.criterion_id)['posters']:
|
for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
|
||||||
addPoster(poster, 'criterion.com', m.criterion_id)
|
addPoster(poster, 'criterion.com', m.criterion_id)
|
||||||
if m.wikipedia_id:
|
if m.wikipedia_id:
|
||||||
poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id)
|
poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
|
||||||
if poster:
|
if poster:
|
||||||
if PosterCache.objects.all().filter(url=poster).count() == 0:
|
if PosterCache.objects.all().filter(url=poster).count() == 0:
|
||||||
addPoster(poster, 'wikipedia.org', m.wikipedia_id)
|
addPoster(poster, 'wikipedia.org', m.wikipedia_id)
|
||||||
if m.impawards_id:
|
if m.impawards_id:
|
||||||
data = oxweb.impawards.getData(m.impawards_id)
|
data = ox.web.impawards.getData(m.impawards_id)
|
||||||
if data and 'imdbId' in data:
|
if data and 'imdbId' in data:
|
||||||
for poster in data['posters']:
|
for poster in data['posters']:
|
||||||
addPoster(poster, 'impawards.com', m.imdb_id)
|
addPoster(poster, 'impawards.com', m.imdb_id)
|
||||||
|
|
||||||
for kg in Karagarga.objects.all().filter(movie_id=m):
|
for kg in Karagarga.objects.all().filter(movie_id=m):
|
||||||
data = oxweb.karagarga.getData(kg.karagarga_id)
|
data = ox.web.karagarga.getData(kg.karagarga_id)
|
||||||
if data:
|
if data:
|
||||||
for poster in data['posters']:
|
for poster in data['posters']:
|
||||||
addPoster(poster, 'karagarga.net', kg.karagarga_id)
|
addPoster(poster, 'karagarga.net', kg.karagarga_id)
|
||||||
|
|
|
@ -7,7 +7,7 @@ from django.contrib.auth.models import User
|
||||||
from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
|
from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
|
||||||
from django.template import RequestContext
|
from django.template import RequestContext
|
||||||
|
|
||||||
from oxdata.utils.shortcuts import render_to_json_response
|
from ox.django.shortcuts import render_to_json_response
|
||||||
from oxdata.lookup.models import MovieId
|
from oxdata.lookup.models import MovieId
|
||||||
|
|
||||||
import models
|
import models
|
||||||
|
|
|
@ -1,113 +0,0 @@
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
# vi:si:et:sw=4:sts=4:ts=4
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
import signal
|
|
||||||
import threading
|
|
||||||
import atexit
|
|
||||||
import Queue
|
|
||||||
|
|
||||||
_interval = 1.0
|
|
||||||
_times = {}
|
|
||||||
_files = []
|
|
||||||
|
|
||||||
_running = False
|
|
||||||
_queue = Queue.Queue()
|
|
||||||
_lock = threading.Lock()
|
|
||||||
|
|
||||||
def _restart(path):
|
|
||||||
_queue.put(True)
|
|
||||||
prefix = 'monitor (pid=%d):' % os.getpid()
|
|
||||||
print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path)
|
|
||||||
print >> sys.stderr, '%s Triggering process restart.' % prefix
|
|
||||||
os.kill(os.getpid(), signal.SIGINT)
|
|
||||||
|
|
||||||
def _modified(path):
|
|
||||||
try:
|
|
||||||
# If path doesn't denote a file and were previously
|
|
||||||
# tracking it, then it has been removed or the file type
|
|
||||||
# has changed so force a restart. If not previously
|
|
||||||
# tracking the file then we can ignore it as probably
|
|
||||||
# pseudo reference such as when file extracted from a
|
|
||||||
# collection of modules contained in a zip file.
|
|
||||||
|
|
||||||
if not os.path.isfile(path):
|
|
||||||
return path in _times
|
|
||||||
|
|
||||||
# Check for when file last modified.
|
|
||||||
|
|
||||||
mtime = os.stat(path).st_mtime
|
|
||||||
if path not in _times:
|
|
||||||
_times[path] = mtime
|
|
||||||
|
|
||||||
# Force restart when modification time has changed, even
|
|
||||||
# if time now older, as that could indicate older file
|
|
||||||
# has been restored.
|
|
||||||
|
|
||||||
if mtime != _times[path]:
|
|
||||||
return True
|
|
||||||
except:
|
|
||||||
# If any exception occured, likely that file has been
|
|
||||||
# been removed just before stat(), so force a restart.
|
|
||||||
|
|
||||||
return True
|
|
||||||
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _monitor():
|
|
||||||
while 1:
|
|
||||||
# Check modification times on all files in sys.modules.
|
|
||||||
|
|
||||||
for module in sys.modules.values():
|
|
||||||
if not hasattr(module, '__file__'):
|
|
||||||
continue
|
|
||||||
path = getattr(module, '__file__')
|
|
||||||
if not path:
|
|
||||||
continue
|
|
||||||
if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']:
|
|
||||||
path = path[:-1]
|
|
||||||
if _modified(path):
|
|
||||||
return _restart(path)
|
|
||||||
|
|
||||||
# Check modification times on files which have
|
|
||||||
# specifically been registered for monitoring.
|
|
||||||
|
|
||||||
for path in _files:
|
|
||||||
if _modified(path):
|
|
||||||
return _restart(path)
|
|
||||||
|
|
||||||
# Go to sleep for specified interval.
|
|
||||||
|
|
||||||
try:
|
|
||||||
return _queue.get(timeout=_interval)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
|
|
||||||
_thread = threading.Thread(target=_monitor)
|
|
||||||
_thread.setDaemon(True)
|
|
||||||
|
|
||||||
def _exiting():
|
|
||||||
try:
|
|
||||||
_queue.put(True)
|
|
||||||
except:
|
|
||||||
pass
|
|
||||||
_thread.join()
|
|
||||||
|
|
||||||
atexit.register(_exiting)
|
|
||||||
|
|
||||||
def track(path):
|
|
||||||
if not path in _files:
|
|
||||||
_files.append(path)
|
|
||||||
|
|
||||||
def start(interval=1.0):
|
|
||||||
global _interval
|
|
||||||
if interval < _interval:
|
|
||||||
_interval = interval
|
|
||||||
|
|
||||||
global _running
|
|
||||||
_lock.acquire()
|
|
||||||
if not _running:
|
|
||||||
_running = True
|
|
||||||
_thread.start()
|
|
||||||
_lock.release()
|
|
|
@ -1,9 +0,0 @@
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
# vi:si:et:sw=4:sts=4:ts=4
|
|
||||||
|
|
||||||
from django.http import HttpResponse
|
|
||||||
from django.utils import simplejson
|
|
||||||
|
|
||||||
def render_to_json_response(dictionary, content_type="text/json"):
|
|
||||||
return HttpResponse(simplejson.dumps(dictionary), content_type=content_type)
|
|
||||||
|
|
|
@ -1,4 +1,6 @@
|
||||||
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
|
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
|
||||||
South==0.6.1
|
South==0.6.1
|
||||||
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxlib
|
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-ox
|
||||||
-e bzr+http://code.0xdb.org/python-oxweb/#egg=python-oxweb
|
-e bzr+http://code.0xdb.org/python-oxweb/#egg=python-oxweb
|
||||||
|
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxdjango
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,7 @@ sys.path.append(root_dir)
|
||||||
sys.path.append(os.path.join(root_dir, project_module))
|
sys.path.append(os.path.join(root_dir, project_module))
|
||||||
|
|
||||||
#reload if this django.wsgi gets touched
|
#reload if this django.wsgi gets touched
|
||||||
from utils import monitor
|
from ox.django import monitor
|
||||||
monitor.start(interval=1.0)
|
monitor.start(interval=1.0)
|
||||||
|
|
||||||
monitor.track(os.path.abspath(os.path.dirname(__file__)))
|
monitor.track(os.path.abspath(os.path.dirname(__file__)))
|
||||||
|
|
Loading…
Reference in a new issue