From d4c2fe794fa6738d81462d26530e5673724bd0e6 Mon Sep 17 00:00:00 2001
From: j <0x006A@0x2620.org>
Date: Wed, 21 Mar 2007 19:16:41 +0000
Subject: [PATCH] - sortnames, json api and function and minimal admin
interface - start of admin interface for archives - highlight function
---
oilarchive/controllers.py | 60 ++++++++++++++++++-
oilarchive/cronjobs.py | 29 +++++++++
oilarchive/forms.py | 24 ++++++++
oilarchive/model.py | 54 +++++++++++++++--
oilarchive/oilspider.py | 24 ++++++++
oilarchive/sortname.py | 33 ++++++++++
.../2007-03-20/ArchiveItem_mysql.sql | 21 -------
.../2007-03-20/Group_mysql.sql | 17 ------
.../2007-03-20/Permission_mysql.sql | 9 ---
.../2007-03-20/User_mysql.sql | 12 ----
.../2007-03-20/VisitIdentity_mysql.sql | 9 ---
.../2007-03-20/Visit_mysql.sql | 10 ----
oilarchive/templates/admin_archives.kid | 18 ++++++
oilarchive/templates/admin_index.kid | 11 ++++
oilarchive/templates/admin_sortnames.kid | 12 ++++
oilarchive/utils.py | 31 ++++++++++
16 files changed, 290 insertions(+), 84 deletions(-)
create mode 100644 oilarchive/cronjobs.py
create mode 100644 oilarchive/forms.py
create mode 100644 oilarchive/oilspider.py
create mode 100644 oilarchive/sortname.py
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/ArchiveItem_mysql.sql
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/Group_mysql.sql
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/Permission_mysql.sql
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/User_mysql.sql
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/VisitIdentity_mysql.sql
delete mode 100644 oilarchive/sqlobject-history/2007-03-20/Visit_mysql.sql
create mode 100644 oilarchive/templates/admin_archives.kid
create mode 100644 oilarchive/templates/admin_index.kid
create mode 100644 oilarchive/templates/admin_sortnames.kid
create mode 100644 oilarchive/utils.py
diff --git a/oilarchive/controllers.py b/oilarchive/controllers.py
index c3eebbb..97fe1bd 100644
--- a/oilarchive/controllers.py
+++ b/oilarchive/controllers.py
@@ -2,7 +2,7 @@
# -*- coding: utf-8 -*-
# vi:si:et:sw=2:sts=2:ts=2
-from turbogears import controllers, expose
+from turbogears import controllers, expose, validate, error_handler
from model import *
from turbogears import identity, redirect
from cherrypy import request, response
@@ -11,7 +11,8 @@ from cherrypy import request, response
# from oilarchive import json
import oilcache
-
+from forms import forms
+from sortname import sortname
class View:
@expose(template=".templates.view")
@@ -36,9 +37,64 @@ class View:
except:
raise redirect("/")
+class Admin:
+ @expose('.templates.admin_index')
+ def index(self):
+ return dict()
+
+ @expose('.templates.admin_archives')
+ def archives(self, tg_errors=None):
+ if tg_errors:
+ flash("There was a problem with the form!")
+ return dict(
+ add_archive = forms.add_archive,
+ archives = Archive.select(orderBy = "archiveName"),
+ )
+
+ @expose()
+ @validate(form = forms.add_archive)
+ @error_handler(archives)
+ def archives_add(self, **data):
+ new = Archive(
+ archiveName = data['archiveName'],
+ archiveType = data['archiveType'],
+ archiveUrl = data['archiveUrl'],
+ ttl = int(data['ttl']),
+ )
+ raise redirect('archives')
+
+ @expose('.templates.admin_sortnames')
+ def sortnames(self, tg_errors=None):
+ if tg_errors:
+ flash("There was a problem with the form!")
+ q = SortName.select(orderBy="name")
+ names = "\n".join([i.name for i in q])
+ sortnames_values = dict(names = names)
+ return dict(sortnames_values = sortnames_values, sortnames_form = forms.sortnames)
+
+ @expose()
+ @validate(form = forms.sortnames)
+ @error_handler(sortnames)
+ def sortnames_save(self, **data):
+ names = data['names']
+ if names:
+ for b in SortName.select():
+ SortName.delete(b.id)
+ names = names.split('\n')
+ for name in names:
+ SortName(name = name)
+ raise redirect('sortnames')
+
+class Api:
+ @expose()
+ def sortname(self, name):
+ sname = sortname(name)
+ return dict(sortname = sname, name = name)
class Root(controllers.RootController):
view = View()
+ admin = Admin()
+ api = Api()
@expose(template=".templates.welcome")
# @identity.require(identity.in_group("admin"))
diff --git a/oilarchive/cronjobs.py b/oilarchive/cronjobs.py
new file mode 100644
index 0000000..402130c
--- /dev/null
+++ b/oilarchive/cronjobs.py
@@ -0,0 +1,29 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from datetime import datetime, timedelta
+
+from model import *
+from sortname import sortname
+
+
+'''
+ update authorSort for better(tm) sorting
+'''
+def updateSortAuthorNames():
+ for i in ArchiveItems.select():
+ i.authorSort = sortname(i.author)
+
+'''
+ grab new input from archives
+'''
+def spiderArchives():
+ for archive in Archives.select():
+ if archive.pubDate - datetime.now() < timedelta(minutes = archive.ttl):
+ archive.update()
+
+
+def runCron():
+ spiderArchives()
+ updateSortAuthorNames()
\ No newline at end of file
diff --git a/oilarchive/forms.py b/oilarchive/forms.py
new file mode 100644
index 0000000..cb59dbc
--- /dev/null
+++ b/oilarchive/forms.py
@@ -0,0 +1,24 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from turbogears import widgets
+
+class forms:
+ sortnames = widgets.TableForm(
+ fields=[
+ widgets.TextArea(name="names",label="Fixed Sortnames")
+ ],
+ submit_text="Save"
+ )
+
+ add_archive = widgets.TableForm(
+ fields=[
+ widgets.TextField(name="archiveName",label="Name"),
+ widgets.TextField(name="archiveType",label="Type"),
+ widgets.TextField(name="archiveUrl",label="JSON url"),
+ widgets.TextField(name="ttl",label="Update Interval(Minutes)"),
+
+ ],
+ submit_text="Save"
+ )
diff --git a/oilarchive/model.py b/oilarchive/model.py
index 9c31804..dad14e1 100644
--- a/oilarchive/model.py
+++ b/oilarchive/model.py
@@ -7,6 +7,10 @@ from datetime import datetime
from turbogears.database import PackageHub
from sqlobject import *
from turbogears import identity
+from scrapeit import read_url
+import simplejson
+
+from oilspider import jsonLoadArchiveItem, jsonPrepareArchiveItem, jsonImportArchiveItem
hub = PackageHub("oilarchive")
__connection__ = hub
@@ -14,9 +18,11 @@ __connection__ = hub
class ArchiveItem(SQLObject):
hashId = UnicodeCol(alternateID = True, length=128)
+ archiveId = UnicodeCol()
title = UnicodeCol()
description = UnicodeCol()
author = UnicodeCol()
+ authorSort = UnicodeCol(default = '')
text = UnicodeCol() #Fulltext
url = UnicodeCol()
downloadURL = UnicodeCol()
@@ -27,7 +33,13 @@ class ArchiveItem(SQLObject):
rights = IntCol() #-> int: 0 (free) - 5 (unfree)
archiveName = UnicodeCol()
archiveType = UnicodeCol()
+ created = DateTimeCol(default=datetime.now)
+ def _set_author(self, value):
+ self._SO_set_author(value)
+ if not self.author_sort:
+ self.author_sort = value
+
def _get_year(self):
return self.releaseDate.strftime('%Y')
@@ -42,12 +54,46 @@ class ArchiveItem(SQLObject):
pubDate = self.pubDate,
size = self.size,
)
+
+ def update(self, data):
+ for key in data:
+ setattr(self, key, values[key])
-def Archive(SQLObject):
- archiveName = UnicodeCol()
- url = UnicodeCol()
- archiveType = UnicodeCol()
+
+class Archive(SQLObject):
+ archiveName = UnicodeCol(alternateID = True, length = 1000)
+ archiveUrl = UnicodeCol()
+ archiveType = UnicodeCol(default=u'')
+ ttl = IntCol(default = "15")
+ pubDate = DateTimeCol(default=datetime.now)
+ created = DateTimeCol(default=datetime.now)
+
+ def _get_pubDateTimestamp(self):
+ return time.mktime(self.pubDate.timetuple())
+ def _get_update_url(self):
+ return "%s?pubDate=%s" % (self.archiveUrl, self.pubDateTimestamp)
+
+ def data_url(self, id):
+ return "%s?id=%s" % (self.archiveUrl, id)
+
+ def update(self):
+ result = simplejson.loads(read_url(self.update_url))
+ for id in result:
+ data = jsonLoadArchiveItem(read_url(self.data_url(id)))
+ q = ArchiveItem.select(AND(
+ ArchiveItem.q.ArchiveId == id,
+ ArchiveItem.q.ArchiveName == self.ArchiveName))
+ if q.count() == 0:
+ data = jsonPrepareArchiveItem(id, data)
+ jsonImportArchiveItem(data)
+ else:
+ q[0].update(data)
+
+
+class SortName(SQLObject):
+ name =UnicodeCol(length=1000, alternateID=True)
+
# identity models.
class Visit(SQLObject):
class sqlmeta:
diff --git a/oilarchive/oilspider.py b/oilarchive/oilspider.py
new file mode 100644
index 0000000..59ff67c
--- /dev/null
+++ b/oilarchive/oilspider.py
@@ -0,0 +1,24 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+import simplejson
+
+from model import *
+
+
+def jsonLoadArchiveItem(data):
+ json_array = simplejson.loads(data)
+ for key in ('releaseDate', 'pubDate'):
+ json_array[key] = datetime.utcfromtimestamp(float(json_array[key]))
+ for key in ('rights', 'size'):
+ json_array[key] = int(json_array[key])
+
+def jsonPrepareArchiveItem(sid, json_array):
+ json_array['archiveId'] = sid
+ return json_array
+
+def jsonImportArchiveItem(archiveId, json_array):
+ json_array = jsonPrepareArchiveItem(archiveId, json_array)
+ ArchiveItem( **json_array)
+
diff --git a/oilarchive/sortname.py b/oilarchive/sortname.py
new file mode 100644
index 0000000..fbe92a9
--- /dev/null
+++ b/oilarchive/sortname.py
@@ -0,0 +1,33 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from model import SortName
+
+
+articles=('the', 'der', 'die', 'das', 'le', 'la', "l'")
+
+'''
+ internal function to generate the sortname, only called if name not in
+ blacklist
+'''
+def _sortname(name):
+ names = name.split(' ')
+ len_names = len(names)
+ if names[0].lower() in articles:
+ names.append(names.pop(0))
+ len_names -= 1
+ if len_names > 1:
+ last_name = names.pop(len_names - 1)
+ names = (last_name, " ".join(names))
+ return ", ".join(names)
+
+'''
+ returns a sortname for name
+'''
+def sortname(name):
+ q = SortName.select(SortName.q.name == name)
+ if q.count() == 0:
+ return _sortname(name)
+ else:
+ return name
\ No newline at end of file
diff --git a/oilarchive/sqlobject-history/2007-03-20/ArchiveItem_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/ArchiveItem_mysql.sql
deleted file mode 100644
index 436c6cf..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/ArchiveItem_mysql.sql
+++ /dev/null
@@ -1,21 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.ArchiveItem
--- Database: mysql
-CREATE TABLE archive_item (
- id INT PRIMARY KEY AUTO_INCREMENT,
- hash_id VARCHAR(128) NOT NULL UNIQUE,
- title TEXT,
- description TEXT,
- author TEXT,
- text TEXT,
- url TEXT,
- download_ur_l TEXT,
- icon TEXT,
- release_date DATETIME,
- pub_date DATETIME,
- size INT,
- rights INT,
- archive_name TEXT,
- archive_type TEXT
-);
-;
diff --git a/oilarchive/sqlobject-history/2007-03-20/Group_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/Group_mysql.sql
deleted file mode 100644
index eb9e3a0..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/Group_mysql.sql
+++ /dev/null
@@ -1,17 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Group
--- Database: mysql
-CREATE TABLE tg_group (
- id INT PRIMARY KEY AUTO_INCREMENT,
- group_name VARCHAR(16) NOT NULL UNIQUE,
- display_name VARCHAR(255),
- created DATETIME
-);
-CREATE TABLE user_group (
-group_id INT NOT NULL,
-user_id INT NOT NULL
-);
-CREATE TABLE group_permission (
-group_id INT NOT NULL,
-permission_id INT NOT NULL
-);
diff --git a/oilarchive/sqlobject-history/2007-03-20/Permission_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/Permission_mysql.sql
deleted file mode 100644
index cd767f2..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/Permission_mysql.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Permission
--- Database: mysql
-CREATE TABLE permission (
- id INT PRIMARY KEY AUTO_INCREMENT,
- permission_name VARCHAR(16) NOT NULL UNIQUE,
- description VARCHAR(255)
-);
-;
diff --git a/oilarchive/sqlobject-history/2007-03-20/User_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/User_mysql.sql
deleted file mode 100644
index 4c41eb4..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/User_mysql.sql
+++ /dev/null
@@ -1,12 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.User
--- Database: mysql
-CREATE TABLE tg_user (
- id INT PRIMARY KEY AUTO_INCREMENT,
- user_name VARCHAR(16) NOT NULL UNIQUE,
- email_address VARCHAR(255) NOT NULL UNIQUE,
- display_name VARCHAR(255),
- password VARCHAR(40),
- created DATETIME
-);
-;
diff --git a/oilarchive/sqlobject-history/2007-03-20/VisitIdentity_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/VisitIdentity_mysql.sql
deleted file mode 100644
index 2444e92..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/VisitIdentity_mysql.sql
+++ /dev/null
@@ -1,9 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.VisitIdentity
--- Database: mysql
-CREATE TABLE visit_identity (
- id INT PRIMARY KEY AUTO_INCREMENT,
- visit_key VARCHAR(40) NOT NULL UNIQUE,
- user_id INT
-);
-;
diff --git a/oilarchive/sqlobject-history/2007-03-20/Visit_mysql.sql b/oilarchive/sqlobject-history/2007-03-20/Visit_mysql.sql
deleted file mode 100644
index c19c603..0000000
--- a/oilarchive/sqlobject-history/2007-03-20/Visit_mysql.sql
+++ /dev/null
@@ -1,10 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Visit
--- Database: mysql
-CREATE TABLE visit (
- id INT PRIMARY KEY AUTO_INCREMENT,
- visit_key VARCHAR(40) NOT NULL UNIQUE,
- created DATETIME,
- expiry DATETIME
-);
-;
diff --git a/oilarchive/templates/admin_archives.kid b/oilarchive/templates/admin_archives.kid
new file mode 100644
index 0000000..3b1e186
--- /dev/null
+++ b/oilarchive/templates/admin_archives.kid
@@ -0,0 +1,18 @@
+
+
+
+
+Oil Archive
+
+
+ known archives
+
+ - ${archive.archiveName}
+
+ add new archive:
+ ${add_archive(action="archives_add")}
+
+ Fill me
+
+
\ No newline at end of file
diff --git a/oilarchive/templates/admin_index.kid b/oilarchive/templates/admin_index.kid
new file mode 100644
index 0000000..16d5324
--- /dev/null
+++ b/oilarchive/templates/admin_index.kid
@@ -0,0 +1,11 @@
+
+
+
+
+Oil Archive
+
+
+ Fill me
+
+
diff --git a/oilarchive/templates/admin_sortnames.kid b/oilarchive/templates/admin_sortnames.kid
new file mode 100644
index 0000000..197d7d9
--- /dev/null
+++ b/oilarchive/templates/admin_sortnames.kid
@@ -0,0 +1,12 @@
+
+
+
+
+Oil Archive
+
+
+ SortNames
+ ${sortnames_form( sortnames_values, action="sortnames_save")}
+
+
diff --git a/oilarchive/utils.py b/oilarchive/utils.py
new file mode 100644
index 0000000..24df6e8
--- /dev/null
+++ b/oilarchive/utils.py
@@ -0,0 +1,31 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+
+'''
+ highlight search term in text, scipping html tags and script elements
+'''
+def highlightText(text, term):
+ highlightStart = u''
+ highlightEnd = u''
+ output = u''
+ if term.strip():
+ term = term.lower()
+ textLower = text.lower()
+ termLength = len(term)
+ while text:
+ i = textLower.find(term)
+ if i == -1:
+ output += text
+ break
+ if textLower[:i].rfind('<') <= textLower[:i].rfind('>') and \
+ textLower[:i].rfind('/script>') >= textLower[:i].rfind('