work around thread issues with ox.cache
This commit is contained in:
parent
07cd885b0a
commit
92d7c210ca
2 changed files with 19 additions and 14 deletions
28
ox/cache.py
28
ox/cache.py
|
@ -140,14 +140,16 @@ class SQLiteCache(Cache):
|
|||
if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
self.db = os.path.join(path, "cache.sqlite")
|
||||
|
||||
def connect(self):
|
||||
self.conn = sqlite3.connect(self.db, timeout=10)
|
||||
self.conn.text_factory = str
|
||||
self.create()
|
||||
|
||||
def connect(self):
|
||||
conn = sqlite3.connect(self.db, timeout=10)
|
||||
conn.text_factory = str
|
||||
return conn
|
||||
|
||||
def create(self):
|
||||
c = self.conn.cursor()
|
||||
conn = self.connect()
|
||||
c = conn.cursor()
|
||||
# Create table and indexes
|
||||
c.execute('''CREATE TABLE IF NOT EXISTS cache (url_hash varchar(42) unique, domain text, url text,
|
||||
post_data text, headers text, created int, data blob, only_headers int)''')
|
||||
|
@ -159,7 +161,7 @@ class SQLiteCache(Cache):
|
|||
if int(self.get_setting(c, 'version', 0)) < 1:
|
||||
self.set_setting(c, 'version', 1)
|
||||
c.execute('''ALTER TABLE cache ADD compressed INT DEFAULT 0''')
|
||||
self.conn.commit()
|
||||
conn.commit()
|
||||
|
||||
def get_setting(self, c, key, default=None):
|
||||
c.execute('SELECT value FROM setting WHERE key = ?', (key, ))
|
||||
|
@ -179,8 +181,8 @@ class SQLiteCache(Cache):
|
|||
else:
|
||||
url_hash = hashlib.sha1(url).hexdigest()
|
||||
|
||||
self.connect()
|
||||
c = self.conn.cursor()
|
||||
conn = self.connect()
|
||||
c = conn.cursor()
|
||||
sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value
|
||||
if timeout > 0:
|
||||
now = time.mktime(time.localtime())
|
||||
|
@ -203,7 +205,7 @@ class SQLiteCache(Cache):
|
|||
break
|
||||
|
||||
c.close()
|
||||
self.conn.close()
|
||||
conn.close()
|
||||
return r
|
||||
|
||||
def set(self, url, post_data, data, headers):
|
||||
|
@ -214,8 +216,8 @@ class SQLiteCache(Cache):
|
|||
|
||||
domain = ".".join(urlparse.urlparse(url)[1].split('.')[-2:])
|
||||
|
||||
self.connect()
|
||||
c = self.conn.cursor()
|
||||
conn = self.connect()
|
||||
c = conn.cursor()
|
||||
|
||||
# Insert a row of data
|
||||
if not post_data: post_data=""
|
||||
|
@ -244,9 +246,9 @@ class SQLiteCache(Cache):
|
|||
c.execute(u"""INSERT OR REPLACE INTO cache values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", t)
|
||||
|
||||
# Save (commit) the changes and clean up
|
||||
self.conn.commit()
|
||||
conn.commit()
|
||||
c.close()
|
||||
self.conn.close()
|
||||
conn.close()
|
||||
|
||||
class FileCache(Cache):
|
||||
def __init__(self):
|
||||
|
|
|
@ -39,7 +39,10 @@ def cmd(program):
|
|||
|
||||
def _get_file_cache():
|
||||
import ox.cache
|
||||
return os.path.join(ox.cache.cache_path(), 'files.sqlite')
|
||||
path = ox.cache.cache_path()
|
||||
if path.startswith('fs:'):
|
||||
path = path[3:]
|
||||
return os.path.join(path, 'files.sqlite')
|
||||
|
||||
def cache(filename, type='oshash'):
|
||||
conn = sqlite3.connect(_get_file_cache(), timeout=10)
|
||||
|
|
Loading…
Reference in a new issue