spider now also collects js/css files and puts the in the template
This commit is contained in:
j
parent
2a6ec2987c
commit
036f03a265
6 changed files with 90 additions and 30 deletions
2
dev.cfg
2
dev.cfg
|
|
@ -5,7 +5,7 @@
|
|||
# oilarchive/config/app.cfg
|
||||
|
||||
# DATABASE
|
||||
sqlobject.dburi="notrans_mysql://root@localhost/oil"
|
||||
sqlobject.dburi="notrans_mysql://root@localhost/oil?sqlobject_encoding=utf-8&use_unicode=1&charset=utf8"
|
||||
|
||||
# SERVER
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
# -*- Mode: Python; -*-
|
||||
# -*- coding: utf-8 -*-
|
||||
# vi:si:et:sw=2:sts=2:ts=2
|
||||
import time
|
||||
|
||||
from turbogears import controllers, expose, validate, error_handler
|
||||
from model import *
|
||||
|
|
@ -99,10 +100,38 @@ class Api:
|
|||
sname = sortname(name)
|
||||
return dict(sortname = sname, name = name)
|
||||
|
||||
|
||||
class ArchiveJavascript:
|
||||
@expose()
|
||||
def default(self, name):
|
||||
name = name.split('.')[0]
|
||||
archive = Archive.byHashId(name)
|
||||
response.headerMap['Content-Type'] = "application/x-javascript"
|
||||
secs = 60*60*24*30
|
||||
secs = 60
|
||||
expires = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + secs))
|
||||
cherrypy.response.headerMap["Expires"] = expires
|
||||
return archive.js
|
||||
|
||||
class ArchiveStyleSheet:
|
||||
@expose()
|
||||
def default(self, name):
|
||||
name = name.split('.')[0]
|
||||
archive = Archive.byHashId(name)
|
||||
response.headerMap['Content-Type'] = "text/css"
|
||||
secs = 60*60*24*30
|
||||
secs = 60
|
||||
expires = cherrypy.lib.httptools.HTTPDate(time.gmtime(time.mktime(time.gmtime()) + secs))
|
||||
cherrypy.response.headerMap["Expires"] = expires
|
||||
return archive.css
|
||||
|
||||
class Root(controllers.RootController):
|
||||
|
||||
view = View()
|
||||
admin = Admin()
|
||||
api = Api()
|
||||
js = ArchiveJavascript()
|
||||
css = ArchiveStyleSheet()
|
||||
|
||||
@expose(template=".templates.login")
|
||||
def login(self, forward_url=None, previous_url=None, *args, **kw):
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@
|
|||
from datetime import datetime
|
||||
import time
|
||||
from urllib import quote
|
||||
import urlparse
|
||||
import md5
|
||||
|
||||
from turbogears.database import PackageHub
|
||||
|
|
@ -118,8 +119,8 @@ class ArchiveItem(SQLObject):
|
|||
self.updateHashID()
|
||||
|
||||
def updateHashID(self):
|
||||
salt = '%s/%s/%s' % (self.archive.archiveName, self.author, self.title)
|
||||
self.hashID = md5.new(salt).hexdigest()
|
||||
salt = u'%s/%s' % (self.archive.archiveName, self.archiveItemId)
|
||||
self.hashID = md5.new(salt.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
class Archive(SQLObject):
|
||||
|
|
@ -130,9 +131,16 @@ class Archive(SQLObject):
|
|||
pubDate = DateTimeCol(default=datetime.now)
|
||||
modDate = DateTimeCol(default=datetime.now)
|
||||
created = DateTimeCol(default=datetime.now)
|
||||
initialized = BoolCol(default = False)
|
||||
css = UnicodeCol(default='')
|
||||
js = UnicodeCol(default='')
|
||||
|
||||
hashId = UnicodeCol(alternateID = True, length=128)
|
||||
|
||||
def _get_pubDateTimestamp(self):
|
||||
if self.initialized:
|
||||
return int(time.mktime(self.pubDate.timetuple()))
|
||||
return -1
|
||||
|
||||
def _query_url(self, query):
|
||||
url = "%s?" % self.archiveUrl
|
||||
|
|
@ -142,6 +150,9 @@ class Archive(SQLObject):
|
|||
def _get_update_url(self):
|
||||
return self._query_url({'modDate': self.pubDateTimestamp})
|
||||
|
||||
def _get_files_url(self):
|
||||
return self._query_url({'files': '1'})
|
||||
|
||||
def data_url(self, id):
|
||||
return self._query_url({'id': id})
|
||||
|
||||
|
|
@ -149,14 +160,26 @@ class Archive(SQLObject):
|
|||
if url.find('://') > 0:
|
||||
return url
|
||||
if url.startswith('/'):
|
||||
url = "%s/%s" % (self.archiveUrl.split('/')[0], url)
|
||||
domain = "://".join(urlparse.urlsplit(self.archiveUrl)[0:2])
|
||||
url = "%s%s" % (domain, url)
|
||||
else:
|
||||
url = "%s/%s" % (self.archiveUrl, url)
|
||||
return url
|
||||
|
||||
def update(self):
|
||||
result = simplejson.loads(read_url(self.files_url))
|
||||
if result.has_key('css'):
|
||||
self.css = read_url(self.full_url(result['css']))
|
||||
else:
|
||||
self.css = ''
|
||||
if result.has_key('js'):
|
||||
self.js = read_url(self.full_url(result['js']))
|
||||
else:
|
||||
self.js = ''
|
||||
result = simplejson.loads(read_url(self.update_url))
|
||||
items = result.get('items', [])
|
||||
for id in items:
|
||||
print "updating / adding ", id
|
||||
data = jsonLoadArchiveItem(read_url(self.data_url(id)))
|
||||
q = ArchiveItem.select(AND(
|
||||
ArchiveItem.q.archiveItemId == id,
|
||||
|
|
@ -165,6 +188,7 @@ class Archive(SQLObject):
|
|||
jsonImportArchiveItem(self, id, data)
|
||||
else:
|
||||
q[0].update(data)
|
||||
self.initialized = True
|
||||
|
||||
'''
|
||||
get list of all items from archive and remove those from ArchiveItem that
|
||||
|
|
@ -174,9 +198,11 @@ class Archive(SQLObject):
|
|||
url = self._query_url({'modDate': -1})
|
||||
result = simplejson.loads(read_url(url))
|
||||
archiveItems = result.get('items', [])
|
||||
archivedItems = [i.archiveItemId for i in ArchiveItem.select(ArchiveItem.q.archiveID == self.id)]
|
||||
removeItems = filter(lambda i: i not in archiveItems, archivedItems)
|
||||
for i in removeItems: ArchiveItem.delete(i)
|
||||
archivedItems = {}
|
||||
for i in ArchiveItem.select(ArchiveItem.q.archiveID == self.id):
|
||||
archivedItems[i.archiveItemId] = i.id
|
||||
removeItems = filter(lambda i: i not in archiveItems, archivedItems.keys())
|
||||
for i in removeItems: ArchiveItem.delete(archivedItems[i])
|
||||
|
||||
class SortName(SQLObject):
|
||||
name =UnicodeCol(length=1000, alternateID=True)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ import md5
|
|||
def jsonLoadArchiveItem(data):
|
||||
json_array = simplejson.loads(data)
|
||||
json_array.pop('tg_flash', None)
|
||||
if json_array.has_key('archiveURL'):
|
||||
json_array['archiveUrl'] = json_array.pop('archiveURL')
|
||||
if json_array.has_key('downloadURL'):
|
||||
json_array['downloadUrl'] = json_array.pop('downloadURL')
|
||||
for key in ('relDate', 'pubDate', 'modDate'):
|
||||
json_array[key] = datetime.utcfromtimestamp(float(json_array[key]))
|
||||
for key in ('rights', 'size'):
|
||||
|
|
@ -24,9 +28,8 @@ def jsonLoadArchiveItem(data):
|
|||
def jsonImportArchiveItem(archive, archiveItemId, json_array):
|
||||
if isinstance(json_array, basestring):
|
||||
json_array = jsonLoadArchiveItem(json_array)
|
||||
salt = '%s/%s/%s' % (archive.archiveName, json_array['author'], json_array['title'])
|
||||
hashID = md5.new(salt).hexdigest()
|
||||
|
||||
salt = u'%s/%s' % (archive.archiveName, archiveItemId)
|
||||
hashID = md5.new(salt.encode('utf-8')).hexdigest()
|
||||
i = model.ArchiveItem(
|
||||
archiveID=archive.id,
|
||||
hashId = hashID,
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@
|
|||
<head py:match="item.tag=='{http://www.w3.org/1999/xhtml}head'" py:attrs="item.items()">
|
||||
<meta content="text/html; charset=UTF-8" http-equiv="content-type" py:replace="''"/>
|
||||
<title py:replace="''">Your title goes here</title>
|
||||
<meta py:replace="item[:]"/>
|
||||
<style type="text/css">
|
||||
#pageLogin
|
||||
{
|
||||
|
|
@ -19,7 +18,7 @@
|
|||
@import "/static/css/archive.css";
|
||||
</style>
|
||||
<script src="/static/js/archive.js" />
|
||||
|
||||
<meta py:replace="item[:]"/>
|
||||
</head>
|
||||
|
||||
<body py:match="item.tag=='{http://www.w3.org/1999/xhtml}body'" py:attrs="item.items()">
|
||||
|
|
|
|||
|
|
@ -3,10 +3,13 @@
|
|||
py:extends="'master.kid'">
|
||||
<head>
|
||||
<meta content="text/html; charset=utf-8" http-equiv="Content-Type" py:replace="''"/>
|
||||
<title>Oil Archive</title>
|
||||
<title>Oil21 - ${item.title}</title>
|
||||
<style type="text/css" media="screen" py:if="item.archive.css">
|
||||
@import "/css/${item.archive.hashId}.css";
|
||||
</style>
|
||||
<script py:if="item.archive.js" src="/js/${item.archive.hashId}.js" />
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div class="item">
|
||||
<img class="itemIcon" src="/view/${item.hashId}/icon.png" />
|
||||
<div class="author">${item.author}</div>
|
||||
|
|
|
|||
Loading…
Reference in a new issue