- sortnames, json api and function and minimal admin interface

- start of admin interface for archives
 - highlight function

oilarchive/controllers.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 # vi:si:et:sw=2:sts=2:ts=2
 
-from turbogears import controllers, expose
+from turbogears import controllers, expose, validate, error_handler
 from model import *
 from turbogears import identity, redirect
 from cherrypy import request, response
@@ -11,7 +11,8 @@ from cherrypy import request, response
 
 # from oilarchive import json
 import oilcache
-
+from forms import forms
+from sortname import sortname
 
 class View:
   @expose(template=".templates.view")
@@ -36,9 +37,64 @@ class View:
     except:
       raise redirect("/")
 
+class Admin:
+  @expose('.templates.admin_index')
+  def index(self):
+    return dict()
+    
+  @expose('.templates.admin_archives')
+  def archives(self, tg_errors=None):
+    if tg_errors:
+      flash("There was a problem with the form!")
+    return dict(
+      add_archive = forms.add_archive, 
+      archives = Archive.select(orderBy = "archiveName"),
+    )
+    
+  @expose()
+  @validate(form = forms.add_archive)
+  @error_handler(archives)
+  def archives_add(self, **data):
+    new = Archive(
+      archiveName = data['archiveName'],
+      archiveType = data['archiveType'],
+      archiveUrl = data['archiveUrl'],
+      ttl = int(data['ttl']),
+      )
+    raise redirect('archives')
+    
+  @expose('.templates.admin_sortnames')
+  def sortnames(self, tg_errors=None):
+    if tg_errors:
+      flash("There was a problem with the form!")
+    q = SortName.select(orderBy="name")
+    names = "\n".join([i.name for i in q])
+    sortnames_values = dict(names = names)
+    return dict(sortnames_values = sortnames_values, sortnames_form = forms.sortnames)
+  
+  @expose()
+  @validate(form = forms.sortnames)
+  @error_handler(sortnames)
+  def sortnames_save(self, **data):
+    names = data['names']
+    if names:
+      for b in SortName.select():
+        SortName.delete(b.id)
+      names = names.split('\n')
+      for name in names:
+        SortName(name = name)
+    raise redirect('sortnames')
+    
+class Api:
+  @expose()
+  def sortname(self, name):
+    sname = sortname(name)
+    return dict(sortname = sname, name = name)
 
 class Root(controllers.RootController):
   view = View()
+  admin = Admin()
+  api = Api()
   
   @expose(template=".templates.welcome")
   # @identity.require(identity.in_group("admin"))

oilarchive/cronjobs.py

@@ -0,0 +1,29 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from datetime import datetime, timedelta
+
+from model import *
+from sortname import sortname
+
+
+'''
+  update authorSort for better(tm) sorting
+'''
+def updateSortAuthorNames():
+  for i in ArchiveItems.select():
+    i.authorSort = sortname(i.author)
+    
+'''
+  grab new input from archives
+'''
+def spiderArchives():
+  for archive in Archives.select():
+    if archive.pubDate - datetime.now() < timedelta(minutes = archive.ttl):
+      archive.update()
+
+
+def runCron():
+  spiderArchives()
+  updateSortAuthorNames()

oilarchive/forms.py

@@ -0,0 +1,24 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from turbogears import widgets
+
+class forms:
+  sortnames = widgets.TableForm(
+      fields=[
+        widgets.TextArea(name="names",label="Fixed Sortnames")
+      ], 
+      submit_text="Save"
+    )
+  
+  add_archive = widgets.TableForm(
+      fields=[
+        widgets.TextField(name="archiveName",label="Name"),
+        widgets.TextField(name="archiveType",label="Type"),
+        widgets.TextField(name="archiveUrl",label="JSON url"),
+        widgets.TextField(name="ttl",label="Update Interval(Minutes)"),
+        
+      ], 
+      submit_text="Save"
+    )

oilarchive/model.py

@@ -7,6 +7,10 @@ from datetime import datetime
 from turbogears.database import PackageHub
 from sqlobject import *
 from turbogears import identity 
+from scrapeit import read_url
+import simplejson
+
+from oilspider import jsonLoadArchiveItem, jsonPrepareArchiveItem, jsonImportArchiveItem
 
 hub = PackageHub("oilarchive")
 __connection__ = hub
@@ -14,9 +18,11 @@ __connection__ = hub
 
 class ArchiveItem(SQLObject):
   hashId = UnicodeCol(alternateID = True, length=128)
+  archiveId = UnicodeCol()
   title = UnicodeCol()
   description = UnicodeCol()
   author = UnicodeCol()
+  authorSort = UnicodeCol(default = '')
   text = UnicodeCol() #Fulltext 
   url = UnicodeCol()
   downloadURL = UnicodeCol()
@@ -27,6 +33,12 @@ class ArchiveItem(SQLObject):
   rights = IntCol() #-> int: 0 (free) - 5 (unfree)
   archiveName = UnicodeCol()
   archiveType = UnicodeCol()
+  created = DateTimeCol(default=datetime.now)
+  
+  def _set_author(self, value):
+    self._SO_set_author(value)
+    if not self.author_sort:
+      self.author_sort = value
       
   def _get_year(self):
     return self.releaseDate.strftime('%Y')
@@ -43,10 +55,44 @@ class ArchiveItem(SQLObject):
       size = self.size,
     )
     
-def Archive(SQLObject):
-  archiveName = UnicodeCol()
-  url = UnicodeCol()
-  archiveType = UnicodeCol()
+  def update(self, data):
+    for key in data:
+       setattr(self, key, values[key])
+
+
+class Archive(SQLObject):
+  archiveName = UnicodeCol(alternateID = True, length = 1000)
+  archiveUrl = UnicodeCol()
+  archiveType = UnicodeCol(default=u'')
+  ttl = IntCol(default = "15")
+  pubDate = DateTimeCol(default=datetime.now)
+  created = DateTimeCol(default=datetime.now)
+
+  def _get_pubDateTimestamp(self):
+    return time.mktime(self.pubDate.timetuple())
+  
+  def _get_update_url(self):
+    return "%s?pubDate=%s" % (self.archiveUrl, self.pubDateTimestamp)
+    
+  def data_url(self, id):
+    return "%s?id=%s" % (self.archiveUrl, id)
+    
+  def update(self):
+    result = simplejson.loads(read_url(self.update_url))
+    for id in result:
+      data = jsonLoadArchiveItem(read_url(self.data_url(id)))
+      q = ArchiveItem.select(AND(
+            ArchiveItem.q.ArchiveId == id, 
+            ArchiveItem.q.ArchiveName == self.ArchiveName))
+      if q.count() == 0:
+        data = jsonPrepareArchiveItem(id, data)
+        jsonImportArchiveItem(data)
+      else:
+        q[0].update(data)
+
+
+class SortName(SQLObject):
+  name =UnicodeCol(length=1000, alternateID=True)
 
 # identity models.
 class Visit(SQLObject):

oilarchive/oilspider.py

@@ -0,0 +1,24 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+import simplejson
+
+from model import *
+
+
+def jsonLoadArchiveItem(data):
+  json_array = simplejson.loads(data)
+  for key in ('releaseDate', 'pubDate'):
+    json_array[key] = datetime.utcfromtimestamp(float(json_array[key]))
+  for key in ('rights', 'size'):
+    json_array[key] = int(json_array[key])
+
+def jsonPrepareArchiveItem(sid, json_array):
+  json_array['archiveId'] = sid
+  return json_array
+
+def jsonImportArchiveItem(archiveId, json_array):
+  json_array = jsonPrepareArchiveItem(archiveId, json_array)
+  ArchiveItem( **json_array)
+

oilarchive/sortname.py

@@ -0,0 +1,33 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+from model import SortName
+
+
+articles=('the', 'der', 'die', 'das', 'le', 'la', "l'")
+
+'''
+  internal function to generate the sortname, only called if name not in 
+  blacklist
+'''
+def _sortname(name):
+  names = name.split(' ')
+  len_names = len(names)
+  if names[0].lower() in articles:
+    names.append(names.pop(0))
+    len_names -= 1
+  if len_names > 1:
+    last_name = names.pop(len_names - 1)
+    names = (last_name, " ".join(names))
+  return ", ".join(names)
+
+'''
+  returns a sortname for name
+'''
+def sortname(name):
+  q = SortName.select(SortName.q.name == name)
+  if q.count() == 0:
+    return _sortname(name)
+  else:
+    return name

oilarchive/sqlobject-history/2007-03-20/ArchiveItem_mysql.sql

@@ -1,21 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.ArchiveItem
--- Database: mysql
-CREATE TABLE archive_item (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    hash_id VARCHAR(128) NOT NULL UNIQUE,
-    title TEXT,
-    description TEXT,
-    author TEXT,
-    text TEXT,
-    url TEXT,
-    download_ur_l TEXT,
-    icon TEXT,
-    release_date DATETIME,
-    pub_date DATETIME,
-    size INT,
-    rights INT,
-    archive_name TEXT,
-    archive_type TEXT
-);
-;

oilarchive/sqlobject-history/2007-03-20/Group_mysql.sql

@@ -1,17 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Group
--- Database: mysql
-CREATE TABLE tg_group (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    group_name VARCHAR(16) NOT NULL UNIQUE,
-    display_name VARCHAR(255),
-    created DATETIME
-);
-CREATE TABLE user_group (
-group_id INT NOT NULL,
-user_id INT NOT NULL
-);
-CREATE TABLE group_permission (
-group_id INT NOT NULL,
-permission_id INT NOT NULL
-);

oilarchive/sqlobject-history/2007-03-20/Permission_mysql.sql

@@ -1,9 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Permission
--- Database: mysql
-CREATE TABLE permission (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    permission_name VARCHAR(16) NOT NULL UNIQUE,
-    description VARCHAR(255)
-);
-;

oilarchive/sqlobject-history/2007-03-20/User_mysql.sql

@@ -1,12 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.User
--- Database: mysql
-CREATE TABLE tg_user (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    user_name VARCHAR(16) NOT NULL UNIQUE,
-    email_address VARCHAR(255) NOT NULL UNIQUE,
-    display_name VARCHAR(255),
-    password VARCHAR(40),
-    created DATETIME
-);
-;

oilarchive/sqlobject-history/2007-03-20/VisitIdentity_mysql.sql

@@ -1,9 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.VisitIdentity
--- Database: mysql
-CREATE TABLE visit_identity (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    visit_key VARCHAR(40) NOT NULL UNIQUE,
-    user_id INT
-);
-;

oilarchive/sqlobject-history/2007-03-20/Visit_mysql.sql

@@ -1,10 +0,0 @@
--- Exported definition from 2007-03-20T17:06:21
--- Class oilarchive.model.Visit
--- Database: mysql
-CREATE TABLE visit (
-    id INT PRIMARY KEY AUTO_INCREMENT,
-    visit_key VARCHAR(40) NOT NULL UNIQUE,
-    created DATETIME,
-    expiry DATETIME
-);
-;

oilarchive/templates/admin_archives.kid

@@ -0,0 +1,18 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://purl.org/kid/ns#"
+    py:extends="'master.kid'">
+<head>
+<meta content="text/html; charset=utf-8" http-equiv="Content-Type" py:replace="''"/>
+<title>Oil Archive</title>
+</head>
+<body>
+  known archives
+  <ul>
+    <li py:for="archive in archives">${archive.archiveName}</li>
+  </ul>
+  add new archive:
+  ${add_archive(action="archives_add")}
+
+  Fill me
+</body>
+</html>

oilarchive/templates/admin_index.kid

@@ -0,0 +1,11 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://purl.org/kid/ns#"
+    py:extends="'master.kid'">
+<head>
+<meta content="text/html; charset=utf-8" http-equiv="Content-Type" py:replace="''"/>
+<title>Oil Archive</title>
+</head>
+<body>
+  Fill me
+</body>
+</html>

oilarchive/templates/admin_sortnames.kid

@@ -0,0 +1,12 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://purl.org/kid/ns#"
+    py:extends="'master.kid'">
+<head>
+<meta content="text/html; charset=utf-8" http-equiv="Content-Type" py:replace="''"/>
+<title>Oil Archive</title>
+</head>
+<body>
+  <h2>SortNames</h2>
+  ${sortnames_form( sortnames_values, action="sortnames_save")}
+</body>
+</html>

oilarchive/utils.py

@@ -0,0 +1,31 @@
+# -*- Mode: Python; -*-
+# -*- coding: utf-8 -*-
+# vi:si:et:sw=2:sts=2:ts=2
+
+
+'''
+  highlight search term in text, scipping html tags and script elements
+'''
+def highlightText(text, term):
+  highlightStart = u'<span class="textHighlight">'
+  highlightEnd = u'</span>'
+  output = u''
+  if term.strip():
+    term = term.lower()
+    textLower = text.lower()
+    termLength = len(term)
+    while text:
+        i = textLower.find(term)
+        if i == -1:
+          output += text
+          break
+        if textLower[:i].rfind('<') <= textLower[:i].rfind('>') and \
+          textLower[:i].rfind('/script>') >= textLower[:i].rfind('<script'): 
+            output += text[:i] + highlightStart + text[i:i+termLength] + highlightEnd
+        else:
+          output += text[:i+termLength]
+        text = text[i+termLength:]
+        textLower = text.lower()
+  else:
+    output = text
+  return output