0x2620/oxdata
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

use ox/ox.web/ox.django

Browse source
This commit is contained in:
j 2009-10-11 15:00:35 +02:00
parent e38385fcd8
commit f7a9ce1364
10 changed files with 40 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
oxdata/lookup/cache.py
View file

@ -3,13 +3,13 @@
import os
from django.conf import settings
from oxlib.cache import getUrlUnicode
from oxlib import findRe
import oxlib.net
import oxweb.criterion
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
from ox.cache import getUrlUnicode
from ox import findRe
import ox.net
import ox.web.criterion
import ox.web.karagarga
import ox.web.imdb
import ox.web.impawards
import models
from oxdata.poster.models import PosterCache
@ -20,10 +20,10 @@ def addPoster(m, url, site, site_id):
p.save()
def getIds():
for id in oxweb.impawards.getIds():
for id in ox.web.impawards.getIds():
if models.MovieId.objects.all().filter(impawards_id=id).count() == 0:
print 'impawards', id
data = oxweb.impawards.getData(id)
data = ox.web.impawards.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.impawards_id:
@ -32,10 +32,10 @@ def getIds():
for poster in data['posters']:
addPoster(m, poster, 'impawards.com', m.imdb_id)
for id in oxweb.criterion.getIds():
for id in ox.web.criterion.getIds():
if models.MovieId.objects.all().filter(criterion_id=id).count() == 0:
print 'criterion', id
data = oxweb.criterion.getData(id)
data = ox.web.criterion.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
if not m.criterion_id:
@ -47,10 +47,10 @@ def getIds():
#kg
lastId = models.Karagarga.maxId()
for id in oxweb.karagarga.getIds(lastId):
for id in ox.web.karagarga.getIds(lastId):
if models.Karagarga.objects.filter(karagarga_id=id).count() == 0:
print 'kg', id
data = oxweb.karagarga.getData(id)
data = ox.web.karagarga.getData(id)
if data and 'imdbId' in data:
m = models.getMovieIdByImdbId(data['imdbId'])
kg = models.Karagarga()

16
oxdata/lookup/models.py
View file

@ -8,9 +8,9 @@ from django.db.models import Q, Max
from django.contrib.auth.models import User
from django.utils import simplejson
import oxweb.wikipedia
import oxweb.imdb
from oxlib import stripTags
import ox.web.wikipedia
import ox.web.imdb
from ox import stripTags
def getMovieIdByImdbId(imdb_id):
@ -56,8 +56,8 @@ class MovieId(models.Model):
def updateFromWikipedia(self):
if self.wikipedia_id:
wikipedia_url = oxweb.wikipedia.getUrl(self.wikipedia_id)
data = oxweb.wikipedia.getMovieData(wikipedia_url)
wikipedia_url = ox.web.wikipedia.getUrl(self.wikipedia_id)
data = ox.web.wikipedia.getMovieData(wikipedia_url)
_key = {}
for key in ('imdb_id', 'amg_id'):
if key in data:
@ -67,7 +67,7 @@ class MovieId(models.Model):
def updateFromImdb(self):
if self.imdb_id:
data = oxweb.imdb.getMovieInfo(self.imdb_id)
data = ox.web.imdb.getMovieInfo(self.imdb_id)
_key = {
'episode title': 'episode_title',
'series title': 'series_title',
@ -76,12 +76,12 @@ class MovieId(models.Model):
if key in data and data[key]:
setattr(self, _key.get(key, key), data[key])
directors = []
credits = oxweb.imdb.getMovieCredits(self.imdb_id)
credits = ox.web.imdb.getMovieCredits(self.imdb_id)
if 'directors' in credits:
directors = credits['directors']
self.director = ', '.join([stripTags(d[0]) for d in directors])
if not self.wikipedia_id:
self.wikipedia_id = oxweb.wikipedia.getId(oxweb.wikipedia.getUrlByImdb(self.imdb_id))
self.wikipedia_id = ox.web.wikipedia.getId(ox.web.wikipedia.getUrlByImdb(self.imdb_id))
if not self.oxdb_id:
self.gen_oxdb_id()
self.save()

2
oxdata/lookup/views.py
View file

@ -5,7 +5,7 @@ from django.db import models
from django.db.models import Q
from django.contrib.auth.models import User
from oxdata.utils.shortcuts import render_to_json_response
from ox.django.shortcuts import render_to_json_response
def ids(request):
json = {}

26
oxdata/poster/models.py
View file

@ -8,11 +8,11 @@ from django.db.models import Q
from django.contrib.auth.models import User
from django.core.files.base import ContentFile
import oxweb.criterion
import oxweb.movieposterdb
import oxweb.karagarga
import oxweb.imdb
import oxweb.impawards
import ox.web.criterion
import ox.web.movieposterdb
import ox.web.karagarga
import ox.web.imdb
import ox.web.impawards
from oxdata.lookup.models import MovieId, Karagarga
@ -56,9 +56,9 @@ class PosterCache(models.Model):
def get(self):
if not self.image and not self.failed:
try:
import oxlib.net
import ox.net
name = hashlib.sha1(self.url).hexdigest()
data = oxlib.net.getUrl(self.url)
data = ox.net.getUrl(self.url)
self.image.save(name, ContentFile(data))
except:
self.failed = True
@ -90,28 +90,28 @@ def getPosterUrls(m):
p.save()
if m.imdb_id:
poster = oxweb.imdb.getMoviePoster(m.imdb_id)
poster = ox.web.imdb.getMoviePoster(m.imdb_id)
if poster:
addPoster(poster, 'imdb.com', m.imdb_id)
for poster in oxweb.movieposterdb.getData(m.imdb_id)['posters']:
for poster in ox.web.movieposterdb.getData(m.imdb_id)['posters']:
addPoster(poster, 'movieposterdb.com', m.imdb_id)
if m.criterion_id:
print 'criterion', m.criterion_id
for poster in oxweb.criterion.getData(m.criterion_id)['posters']:
for poster in ox.web.criterion.getData(m.criterion_id)['posters']:
addPoster(poster, 'criterion.com', m.criterion_id)
if m.wikipedia_id:
poster = oxweb.wikipedia.getPosterUrl(m.wikipedia_id)
poster = ox.web.wikipedia.getPosterUrl(m.wikipedia_id)
if poster:
if PosterCache.objects.all().filter(url=poster).count() == 0:
addPoster(poster, 'wikipedia.org', m.wikipedia_id)
if m.impawards_id:
data = oxweb.impawards.getData(m.impawards_id)
data = ox.web.impawards.getData(m.impawards_id)
if data and 'imdbId' in data:
for poster in data['posters']:
addPoster(poster, 'impawards.com', m.imdb_id)
for kg in Karagarga.objects.all().filter(movie_id=m):
data = oxweb.karagarga.getData(kg.karagarga_id)
data = ox.web.karagarga.getData(kg.karagarga_id)
if data:
for poster in data['posters']:
addPoster(poster, 'karagarga.net', kg.karagarga_id)

2
oxdata/poster/views.py
View file

@ -7,7 +7,7 @@ from django.contrib.auth.models import User
from django.shortcuts import render_to_response, get_object_or_404, get_list_or_404
from django.template import RequestContext
from oxdata.utils.shortcuts import render_to_json_response
from ox.django.shortcuts import render_to_json_response
from oxdata.lookup.models import MovieId
import models

0
oxdata/utils/__init__.py
View file

113
oxdata/utils/monitor.py
View file

@ -1,113 +0,0 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
import os
import sys
import time
import signal
import threading
import atexit
import Queue
_interval = 1.0
_times = {}
_files = []
_running = False
_queue = Queue.Queue()
_lock = threading.Lock()
def _restart(path):
_queue.put(True)
prefix = 'monitor (pid=%d):' % os.getpid()
print >> sys.stderr, '%s Change detected to \'%s\'.' % (prefix, path)
print >> sys.stderr, '%s Triggering process restart.' % prefix
os.kill(os.getpid(), signal.SIGINT)
def _modified(path):
try:
# If path doesn't denote a file and were previously
# tracking it, then it has been removed or the file type
# has changed so force a restart. If not previously
# tracking the file then we can ignore it as probably
# pseudo reference such as when file extracted from a
# collection of modules contained in a zip file.
if not os.path.isfile(path):
return path in _times
# Check for when file last modified.
mtime = os.stat(path).st_mtime
if path not in _times:
_times[path] = mtime
# Force restart when modification time has changed, even
# if time now older, as that could indicate older file
# has been restored.
if mtime != _times[path]:
return True
except:
# If any exception occured, likely that file has been
# been removed just before stat(), so force a restart.
return True
return False
def _monitor():
while 1:
# Check modification times on all files in sys.modules.
for module in sys.modules.values():
if not hasattr(module, '__file__'):
continue
path = getattr(module, '__file__')
if not path:
continue
if os.path.splitext(path)[1] in ['.pyc', '.pyo', '.pyd']:
path = path[:-1]
if _modified(path):
return _restart(path)
# Check modification times on files which have
# specifically been registered for monitoring.
for path in _files:
if _modified(path):
return _restart(path)
# Go to sleep for specified interval.
try:
return _queue.get(timeout=_interval)
except:
pass
_thread = threading.Thread(target=_monitor)
_thread.setDaemon(True)
def _exiting():
try:
_queue.put(True)
except:
pass
_thread.join()
atexit.register(_exiting)
def track(path):
if not path in _files:
_files.append(path)
def start(interval=1.0):
global _interval
if interval < _interval:
_interval = interval
global _running
_lock.acquire()
if not _running:
_running = True
_thread.start()
_lock.release()

9
oxdata/utils/shortcuts.py
View file

@ -1,9 +0,0 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=4:sts=4:ts=4
from django.http import HttpResponse
from django.utils import simplejson
def render_to_json_response(dictionary, content_type="text/json"):
return HttpResponse(simplejson.dumps(dictionary), content_type=content_type)

4
requirements.txt
View file

@ -1,4 +1,6 @@
-e svn+http://code.djangoproject.com/svn/django/branches/releases/1.1.X/#egg=django
South==0.6.1
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxlib
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-ox
-e bzr+http://code.0xdb.org/python-oxweb/#egg=python-oxweb
-e bzr+http://code.0xdb.org/python-oxlib/#egg=python-oxdjango

2
wsgi/django.wsgi
View file

@ -15,7 +15,7 @@ sys.path.append(root_dir)
sys.path.append(os.path.join(root_dir, project_module))
#reload if this django.wsgi gets touched
from utils import monitor
from ox.django import monitor
monitor.start(interval=1.0)
monitor.track(os.path.abspath(os.path.dirname(__file__)))