work around thread issues with ox.cache

ox/cache.py

@@ -140,14 +140,16 @@ class SQLiteCache(Cache):
         if not os.path.exists(path):
             os.makedirs(path)
         self.db = os.path.join(path, "cache.sqlite")
-
-    def connect(self):
-        self.conn = sqlite3.connect(self.db, timeout=10)
-        self.conn.text_factory = str
         self.create()
 
+    def connect(self):
+        conn = sqlite3.connect(self.db, timeout=10)
+        conn.text_factory = str
+        return conn
+
     def create(self):
-        c = self.conn.cursor()
+        conn = self.connect()
+        c = conn.cursor()
         # Create table and indexes 
         c.execute('''CREATE TABLE IF NOT EXISTS cache (url_hash varchar(42) unique, domain text, url text,
                           post_data text, headers text, created int, data blob, only_headers int)''')
@@ -159,7 +161,7 @@ class SQLiteCache(Cache):
         if int(self.get_setting(c, 'version', 0)) < 1:
             self.set_setting(c, 'version', 1)
             c.execute('''ALTER TABLE cache ADD compressed INT DEFAULT 0''')
-            self.conn.commit()
+            conn.commit()
     
     def get_setting(self, c, key, default=None):
         c.execute('SELECT value FROM setting WHERE key = ?', (key, ))
@@ -179,8 +181,8 @@ class SQLiteCache(Cache):
         else:
             url_hash = hashlib.sha1(url).hexdigest()
 
-        self.connect()
-        c = self.conn.cursor()
+        conn = self.connect()
+        c = conn.cursor()
         sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value
         if timeout > 0:
             now = time.mktime(time.localtime())
@@ -203,7 +205,7 @@ class SQLiteCache(Cache):
             break
 
         c.close()
-        self.conn.close()
+        conn.close()
         return r
 
     def set(self, url, post_data, data, headers):
@@ -214,8 +216,8 @@ class SQLiteCache(Cache):
 
         domain = ".".join(urlparse.urlparse(url)[1].split('.')[-2:])
 
-        self.connect()
-        c = self.conn.cursor()
+        conn = self.connect()
+        c = conn.cursor()
 
         # Insert a row of data
         if not post_data: post_data=""
@@ -244,9 +246,9 @@ class SQLiteCache(Cache):
         c.execute(u"""INSERT OR REPLACE INTO cache values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", t)
 
         # Save (commit) the changes and clean up
-        self.conn.commit()
+        conn.commit()
         c.close()
-        self.conn.close()
+        conn.close()
 
 class FileCache(Cache):
     def __init__(self):

ox/file.py

@@ -39,7 +39,10 @@ def cmd(program):
 
 def _get_file_cache():
     import ox.cache
-    return os.path.join(ox.cache.cache_path(), 'files.sqlite')
+    path = ox.cache.cache_path()
+    if path.startswith('fs:'):
+        path = path[3:]
+    return os.path.join(path, 'files.sqlite')
 
 def cache(filename, type='oshash'):
     conn = sqlite3.connect(_get_file_cache(), timeout=10)