0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

work around thread issues with ox.cache

Browse source
This commit is contained in:
j 2014-05-17 11:25:19 +02:00
parent 07cd885b0a
commit 92d7c210ca
2 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
ox/cache.py
View file

@ -140,14 +140,16 @@ class SQLiteCache(Cache):
if not os.path.exists(path): if not os.path.exists(path):
os.makedirs(path) os.makedirs(path)
self.db = os.path.join(path, "cache.sqlite") self.db = os.path.join(path, "cache.sqlite")
def connect(self):
self.conn = sqlite3.connect(self.db, timeout=10)
self.conn.text_factory = str
self.create() self.create()
def connect(self):
conn = sqlite3.connect(self.db, timeout=10)
conn.text_factory = str
return conn
def create(self): def create(self):
c = self.conn.cursor() conn = self.connect()
c = conn.cursor()
# Create table and indexes # Create table and indexes
c.execute('''CREATE TABLE IF NOT EXISTS cache (url_hash varchar(42) unique, domain text, url text, c.execute('''CREATE TABLE IF NOT EXISTS cache (url_hash varchar(42) unique, domain text, url text,
post_data text, headers text, created int, data blob, only_headers int)''') post_data text, headers text, created int, data blob, only_headers int)''')
@ -159,7 +161,7 @@ class SQLiteCache(Cache):
if int(self.get_setting(c, 'version', 0)) < 1: if int(self.get_setting(c, 'version', 0)) < 1:
self.set_setting(c, 'version', 1) self.set_setting(c, 'version', 1)
c.execute('''ALTER TABLE cache ADD compressed INT DEFAULT 0''') c.execute('''ALTER TABLE cache ADD compressed INT DEFAULT 0''')
self.conn.commit() conn.commit()
def get_setting(self, c, key, default=None): def get_setting(self, c, key, default=None):
c.execute('SELECT value FROM setting WHERE key = ?', (key, )) c.execute('SELECT value FROM setting WHERE key = ?', (key, ))
@ -179,8 +181,8 @@ class SQLiteCache(Cache):
else: else:
url_hash = hashlib.sha1(url).hexdigest() url_hash = hashlib.sha1(url).hexdigest()
self.connect() conn = self.connect()
c = self.conn.cursor() c = conn.cursor()
sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value sql = 'SELECT %s, compressed FROM cache WHERE url_hash=?' % value
if timeout > 0: if timeout > 0:
now = time.mktime(time.localtime()) now = time.mktime(time.localtime())
@ -203,7 +205,7 @@ class SQLiteCache(Cache):
break break
c.close() c.close()
self.conn.close() conn.close()
return r return r
def set(self, url, post_data, data, headers): def set(self, url, post_data, data, headers):
@ -214,8 +216,8 @@ class SQLiteCache(Cache):
domain = ".".join(urlparse.urlparse(url)[1].split('.')[-2:]) domain = ".".join(urlparse.urlparse(url)[1].split('.')[-2:])
self.connect() conn = self.connect()
c = self.conn.cursor() c = conn.cursor()
# Insert a row of data # Insert a row of data
if not post_data: post_data="" if not post_data: post_data=""
@ -244,9 +246,9 @@ class SQLiteCache(Cache):
c.execute(u"""INSERT OR REPLACE INTO cache values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", t) c.execute(u"""INSERT OR REPLACE INTO cache values (?, ?, ?, ?, ?, ?, ?, ?, ?)""", t)
# Save (commit) the changes and clean up # Save (commit) the changes and clean up
self.conn.commit() conn.commit()
c.close() c.close()
self.conn.close() conn.close()
class FileCache(Cache): class FileCache(Cache):
def __init__(self): def __init__(self):

5
ox/file.py
View file

@ -39,7 +39,10 @@ def cmd(program):
def _get_file_cache(): def _get_file_cache():
import ox.cache import ox.cache
return os.path.join(ox.cache.cache_path(), 'files.sqlite') path = ox.cache.cache_path()
if path.startswith('fs:'):
path = path[3:]
return os.path.join(path, 'files.sqlite')
def cache(filename, type='oshash'): def cache(filename, type='oshash'):
conn = sqlite3.connect(_get_file_cache(), timeout=10) conn = sqlite3.connect(_get_file_cache(), timeout=10)