diff --git a/dev.cfg b/dev.cfg index 4bb6ed5..c5cfddb 100644 --- a/dev.cfg +++ b/dev.cfg @@ -5,7 +5,7 @@ # oilarchive/config/app.cfg # DATABASE -sqlobject.dburi="notrans_mysql://root@localhost/oil" +sqlobject.dburi="notrans_mysql://root@localhost/oil?sqlobject_encoding=utf-8&use_unicode=1&charset=utf8" # SERVER diff --git a/oilarchive/controllers.py b/oilarchive/controllers.py index 87fdda0..da3c60b 100644 --- a/oilarchive/controllers.py +++ b/oilarchive/controllers.py @@ -1,6 +1,7 @@ # -*- Mode: Python; -*- # -*- coding: utf-8 -*- # vi:si:et:sw=2:sts=2:ts=2 +import time from turbogears import controllers, expose, validate, error_handler from model import * @@ -99,10 +100,38 @@ class Api: sname = sortname(name) return dict(sortname = sname, name = name) + +class ArchiveJavascript: + @expose() + def default(self, name): + name = name.split('.')[0] + archive = Archive.byHashId(name) + response.headerMap['Content-Type'] = "application/x-javascript" + secs = 60*60*24*30 + secs = 60 + expires = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + secs)) + cherrypy.response.headerMap["Expires"] = expires + return archive.js + +class ArchiveStyleSheet: + @expose() + def default(self, name): + name = name.split('.')[0] + archive = Archive.byHashId(name) + response.headerMap['Content-Type'] = "text/css" + secs = 60*60*24*30 + secs = 60 + expires = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + secs)) + cherrypy.response.headerMap["Expires"] = expires + return archive.css + class Root(controllers.RootController): + view = View() admin = Admin() api = Api() + js = ArchiveJavascript() + css = ArchiveStyleSheet() @expose(template=".templates.login") def login(self, forward_url=None, previous_url=None, *args, **kw): diff --git a/oilarchive/model.py b/oilarchive/model.py index 503599b..f170db6 100644 --- a/oilarchive/model.py +++ b/oilarchive/model.py @@ -5,6 +5,7 @@ from datetime import datetime import time from urllib import quote +import urlparse import md5 from turbogears.database import PackageHub @@ -118,8 +119,8 @@ class ArchiveItem(SQLObject): self.updateHashID() def updateHashID(self): - salt = '%s/%s/%s' % (self.archive.archiveName, self.author, self.title) - self.hashID = md5.new(salt).hexdigest() + salt = u'%s/%s' % (self.archive.archiveName, self.archiveItemId) + self.hashID = md5.new(salt.encode('utf-8')).hexdigest() class Archive(SQLObject): @@ -130,10 +131,17 @@ class Archive(SQLObject): pubDate = DateTimeCol(default=datetime.now) modDate = DateTimeCol(default=datetime.now) created = DateTimeCol(default=datetime.now) - - def _get_pubDateTimestamp(self): - return int(time.mktime(self.pubDate.timetuple())) + initialized = BoolCol(default = False) + css = UnicodeCol(default='') + js = UnicodeCol(default='') + hashId = UnicodeCol(alternateID = True, length=128) + + def _get_pubDateTimestamp(self): + if self.initialized: + return int(time.mktime(self.pubDate.timetuple())) + return -1 + def _query_url(self, query): url = "%s?" % self.archiveUrl url += "&".join(["%s=%s" % (key, quote("%s" % query[key])) for key in query]) @@ -142,6 +150,9 @@ class Archive(SQLObject): def _get_update_url(self): return self._query_url({'modDate': self.pubDateTimestamp}) + def _get_files_url(self): + return self._query_url({'files': '1'}) + def data_url(self, id): return self._query_url({'id': id}) @@ -149,14 +160,26 @@ class Archive(SQLObject): if url.find('://') > 0: return url if url.startswith('/'): - url = "%s/%s" % (self.archiveUrl.split('/')[0], url) + domain = "://".join(urlparse.urlsplit(self.archiveUrl)[0:2]) + url = "%s%s" % (domain, url) else: url = "%s/%s" % (self.archiveUrl, url) + return url def update(self): + result = simplejson.loads(read_url(self.files_url)) + if result.has_key('css'): + self.css = read_url(self.full_url(result['css'])) + else: + self.css = '' + if result.has_key('js'): + self.js = read_url(self.full_url(result['js'])) + else: + self.js = '' result = simplejson.loads(read_url(self.update_url)) items = result.get('items', []) for id in items: + print "updating / adding ", id data = jsonLoadArchiveItem(read_url(self.data_url(id))) q = ArchiveItem.select(AND( ArchiveItem.q.archiveItemId == id, @@ -165,6 +188,7 @@ class Archive(SQLObject): jsonImportArchiveItem(self, id, data) else: q[0].update(data) + self.initialized = True ''' get list of all items from archive and remove those from ArchiveItem that @@ -174,9 +198,11 @@ class Archive(SQLObject): url = self._query_url({'modDate': -1}) result = simplejson.loads(read_url(url)) archiveItems = result.get('items', []) - archivedItems = [i.archiveItemId for i in ArchiveItem.select(ArchiveItem.q.archiveID == self.id)] - removeItems = filter(lambda i: i not in archiveItems, archivedItems) - for i in removeItems: ArchiveItem.delete(i) + archivedItems = {} + for i in ArchiveItem.select(ArchiveItem.q.archiveID == self.id): + archivedItems[i.archiveItemId] = i.id + removeItems = filter(lambda i: i not in archiveItems, archivedItems.keys()) + for i in removeItems: ArchiveItem.delete(archivedItems[i]) class SortName(SQLObject): name =UnicodeCol(length=1000, alternateID=True) diff --git a/oilarchive/oilspider.py b/oilarchive/oilspider.py index 2e2c97a..4dec838 100644 --- a/oilarchive/oilspider.py +++ b/oilarchive/oilspider.py @@ -13,6 +13,10 @@ import md5 def jsonLoadArchiveItem(data): json_array = simplejson.loads(data) json_array.pop('tg_flash', None) + if json_array.has_key('archiveURL'): + json_array['archiveUrl'] = json_array.pop('archiveURL') + if json_array.has_key('downloadURL'): + json_array['downloadUrl'] = json_array.pop('downloadURL') for key in ('relDate', 'pubDate', 'modDate'): json_array[key] = datetime.utcfromtimestamp(float(json_array[key])) for key in ('rights', 'size'): @@ -24,9 +28,8 @@ def jsonLoadArchiveItem(data): def jsonImportArchiveItem(archive, archiveItemId, json_array): if isinstance(json_array, basestring): json_array = jsonLoadArchiveItem(json_array) - salt = '%s/%s/%s' % (archive.archiveName, json_array['author'], json_array['title']) - hashID = md5.new(salt).hexdigest() - + salt = u'%s/%s' % (archive.archiveName, archiveItemId) + hashID = md5.new(salt.encode('utf-8')).hexdigest() i = model.ArchiveItem( archiveID=archive.id, hashId = hashID, diff --git a/oilarchive/templates/master.kid b/oilarchive/templates/master.kid index fc524f6..cc2f613 100644 --- a/oilarchive/templates/master.kid +++ b/oilarchive/templates/master.kid @@ -5,21 +5,20 @@ Your title goes here - - + - -