work around thread issues with ox.cache

This commit is contained in:
j 2014-05-17 11:25:19 +02:00
parent 07cd885b0a
commit 92d7c210ca
2 changed files with 19 additions and 14 deletions

View file

@ -140,14 +140,16 @@ class SQLiteCache(Cache):
if not os.path.exists(path):
os.makedirs(path)
self.db = os.path.join(path, "cache.sqlite")
def connect(self):
self.conn = sqlite3.connect(self.db, timeout=10)
self.conn.text_factory = str
self.create()
def connect(self):
conn = sqlite3.connect(self.db, timeout=10)
conn.text_factory = str
return conn
def create(self):
c = self.conn.cursor()
conn = self.connect()
c = conn.cursor()
# Create table and indexes
c.execute('''CREATE TABLE IF NOT EXISTS cache (url_hash varchar(42) unique, domain text, url text,
post_data text, headers text, created int, data blob, only_headers int)''')
@ -159,7 +161,7 @@ class SQLiteCache(Cache):
if int(self.get_setting(c, 'version', 0)) < 1:
self.set_setting(c, 'version', 1)
c.execute('''ALTER TABLE cache ADD compressed INT DEFAULT 0''')
self.conn.commit()
conn.commit()
def get_setting(self, c, key, default=None):
c.execute('SELECT value FROM setting WHERE key = ?', (key, ))
@ -179,8 +181,8 @@ class SQLiteCache(Cache):
else:
url_hash = hashlib.sha1(url).hexdigest()
self.connect()
c = self.conn.cursor()
conn = self.connect()
c = conn.cursor()
sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value
if timeout > 0:
now = time.mktime(time.localtime())
@ -203,7 +205,7 @@ class SQLiteCache(Cache):
break
c.close()
self.conn.close()
conn.close()
return r
def set(self, url, post_data, data, headers):
@ -214,8 +216,8 @@ class SQLiteCache(Cache):
domain = ".".join(urlparse.urlparse(url)[1].split('.')[-2:])
self.connect()
c = self.conn.cursor()
conn = self.connect()
c = conn.cursor()
# Insert a row of data
if not post_data: post_data=""
@ -244,9 +246,9 @@ class SQLiteCache(Cache):
c.execute(u"""INSERT OR REPLACE INTO cache values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", t)
# Save (commit) the changes and clean up
self.conn.commit()
conn.commit()
c.close()
self.conn.close()
conn.close()
class FileCache(Cache):
def __init__(self):

View file

@ -39,7 +39,10 @@ def cmd(program):
def _get_file_cache():
import ox.cache
return os.path.join(ox.cache.cache_path(), 'files.sqlite')
path = ox.cache.cache_path()
if path.startswith('fs:'):
path = path[3:]
return os.path.join(path, 'files.sqlite')
def cache(filename, type='oshash'):
conn = sqlite3.connect(_get_file_cache(), timeout=10)