0x2620/scrapeit
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

and do not fail for new files

Browse source
This commit is contained in:
j 2008-03-19 13:58:48 +00:00
parent cf955e4e9f
commit 963d95ec0d
1 changed files with 17 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
scrapeit/cache.py
View file

@ -22,24 +22,23 @@ def read_url(url):
cache_file = "%sindex.html" % cache_file
if os.path.isdir(cache_file):
cache_file = os.path.join(cache_file, "index.html")
ctime = os.stat(cache_file).st_ctime
now = time.mktime(time.localtime())
file_age = now-ctime
print cache_timeout-file_age
if file_age < cache_timeout and os.path.exists(cache_file):
f = open(cache_file)
data = f.read()
f.close()
return data
else:
data = utils.read_url(url)
folder = os.path.dirname(cache_file)
if not os.path.exists(folder):
os.makedirs(folder)
f = open(cache_file, 'w')
f.write(data)
f.close()
return data
if os.path.exists(cache_file):
ctime = os.stat(cache_file).st_ctime
now = time.mktime(time.localtime())
file_age = now-ctime
if file_age < cache_timeout:
f = open(cache_file)
data = f.read()
f.close()
return data
data = utils.read_url(url)
folder = os.path.dirname(cache_file)
if not os.path.exists(folder):
os.makedirs(folder)
f = open(cache_file, 'w')
f.write(data)
f.close()
return data
def read_url_utf8(url):
data = read_url(url)