0x2620/python-ox
2
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

net/cache readUrl->read_url / Unicode -> unicode=True

Browse source 
format replace all CammelCase with under_score
This commit is contained in:
j 2012-08-14 15:58:05 +02:00
commit 2de989e188
33 changed files with 243 additions and 254 deletions
Download patch file Download diff file Expand all files Collapse all files

26
ox/web/allmovie.py
View file

@ -3,8 +3,8 @@
import re
import time
from ox import stripTags, findRe
from ox.cache import readUrlUnicode
from ox import strip_tags, findRe
from ox.cache import read_url
def getId(url):
@ -26,7 +26,7 @@ def getData(id):
data = {
"url": getUrl(id)
}
html = readUrlUnicode(data["url"])
html = read_url(data["url"], unicode=True)
data['aka'] = parseList(html, 'AKA')
data['category'] = findRe(html, '<dt>category</dt>.*?<dd>(.*?)</dd>')
data['countries'] = parseList(html, 'countries')
@ -40,18 +40,18 @@ def getData(id):
data['releasedate'] = parseList(html, 'release date')
data['runtime'] = parseEntry(html, 'run time').replace('min.', '').strip()
data['set'] = parseEntry(html, 'set in')
data['synopsis'] = stripTags(findRe(html, '<div class="toggle-text" itemprop="description">(.*?)</div>')).strip()
data['synopsis'] = strip_tags(findRe(html, '<div class="toggle-text" itemprop="description">(.*?)</div>')).strip()
data['themes'] = parseList(html, 'themes')
data['types'] = parseList(html, 'types')
data['year'] = findRe(html, '<span class="year">.*?(\d+)')
#data['stills'] = [re.sub('_derived.*?/', '', i) for i in re.compile('<a href="#" title="movie still".*?<img src="(.*?)"', re.DOTALL).findall(html)]
data['stills'] = re.compile('<a href="#" title="movie still".*?<img src="(.*?)"', re.DOTALL).findall(html)
#html = readUrlUnicode("http://allmovie.com/work/%s/cast" % id)
#html = read_url("http://allmovie.com/work/%s/cast" % id, unicode=True)
#data['cast'] = parseTable(html)
#html = readUrlUnicode("http://allmovie.com/work/%s/credits" % id)
#html = read_url("http://allmovie.com/work/%s/credits" % id, unicode=True)
#data['credits'] = parseTable(html)
html = readUrlUnicode("http://allmovie.com/work/%s/review" % id)
data['review'] = stripTags(findRe(html, '<div class="toggle-text" itemprop="description">(.*?)</div>')).strip()
html = read_url("http://allmovie.com/work/%s/review" % id, unicode=True)
data['review'] = strip_tags(findRe(html, '<div class="toggle-text" itemprop="description">(.*?)</div>')).strip()
return data
def getUrl(id):
@ -59,26 +59,26 @@ def getUrl(id):
def parseEntry(html, title):
html = findRe(html, '<dt>%s</dt>.*?<dd>(.*?)</dd>' % title)
return stripTags(html).strip()
return strip_tags(html).strip()
def parseList(html, title):
html = findRe(html, '<dt>%s</dt>.*?<dd>(.*?)</dd>' % title.lower())
r = map(lambda x: stripTags(x), re.compile('<li>(.*?)</li>', re.DOTALL).findall(html))
r = map(lambda x: strip_tags(x), re.compile('<li>(.*?)</li>', re.DOTALL).findall(html))
if not r and html:
r = [stripTags(html)]
r = [strip_tags(html)]
return r
def parseTable(html):
return map(
lambda x: map(
lambda x: stripTags(x).strip().replace('&nbsp;', ''),
lambda x: strip_tags(x).strip().replace('&nbsp;', ''),
x.split('<td width="305">-')
),
findRe(html, '<div id="results-table">(.*?)</table>').split('</tr>')[:-1]
)
def parseText(html, title):
return stripTags(findRe(html, '%s</td>.*?<td colspan="2"><p>(.*?)</td>' % title)).strip()
return strip_tags(findRe(html, '%s</td>.*?<td colspan="2"><p>(.*?)</td>' % title)).strip()
if __name__ == '__main__':
print getData('129689')