0x2620/scrapeit
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

lets try utf-8

Browse source
This commit is contained in:
j 2007-06-22 15:09:34 +00:00
parent ec63d9fb7c
commit f386b7daba
1 changed files with 9 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
scrapeit/imdb.py
View file

@ -233,7 +233,7 @@ class IMDb:
def getCredits(self, forcereload = False): def getCredits(self, forcereload = False):
if forcereload or not self.creditsSource: if forcereload or not self.creditsSource:
self.creditsSource = read_url(self.creditsUrl) self.creditsSource = read_url_utf8(self.creditsUrl)
return self.creditsSource return self.creditsSource
def parseCredits(self): def parseCredits(self):
@ -261,14 +261,14 @@ class IMDb:
def getPlot(self, forcereload = False): def getPlot(self, forcereload = False):
if forcereload or not self.plotSource: if forcereload or not self.plotSource:
self.plotSource = read_url(self.plotUrl) self.plotSource = read_url_utf8(self.plotUrl)
return self.plotSource return self.plotSource
def parsePlot(self): def parsePlot(self):
soup = BeautifulSoup(self.getPlot()) soup = BeautifulSoup(self.getPlot())
plot = soup('p', {'class':'plotpar'}) plot = soup('p', {'class':'plotpar'})
if plot: if plot:
plot = str(plot[0]).split('<i>')[0] plot = unicode(plot[0]).split('<i>')[0]
else: else:
plot = u'' plot = u''
plot = stripTags(plot).strip() plot = stripTags(plot).strip()
@ -277,7 +277,7 @@ class IMDb:
def getEpisodes(self, forcereload = False): def getEpisodes(self, forcereload = False):
if forcereload or not self.episodesSource: if forcereload or not self.episodesSource:
self.episodesSource = read_url(self.episodesUrl) self.episodesSource = read_url_utf8(self.episodesUrl)
return self.episodesSource return self.episodesSource
def parseEpisodes(self): def parseEpisodes(self):
@ -305,7 +305,7 @@ class IMDb:
def getKeywords(self, forcereload = False): def getKeywords(self, forcereload = False):
if forcereload or not self.keywordSource: if forcereload or not self.keywordSource:
self.keywordSource = read_url(self.keywordUrl) self.keywordSource = read_url_utf8(self.keywordUrl)
return self.keywordSource return self.keywordSource
def parseKeywords(self): def parseKeywords(self):
@ -318,7 +318,7 @@ class IMDb:
def getTrivia(self, forcereload = False): def getTrivia(self, forcereload = False):
if forcereload or not self.triviaSource: if forcereload or not self.triviaSource:
self.triviaSource = read_url(self.triviaUrl) self.triviaSource = read_url_utf8(self.triviaUrl)
return self.triviaSource return self.triviaSource
def parseTrivia(self): def parseTrivia(self):
@ -336,7 +336,7 @@ class IMDb:
def getConnections(self, forcereload = False): def getConnections(self, forcereload = False):
if forcereload or not self.connectionsSource: if forcereload or not self.connectionsSource:
self.connectionsSource = read_url(self.connectionsUrl) self.connectionsSource = read_url_utf8(self.connectionsUrl)
return self.connectionsSource return self.connectionsSource
def parseConnections(self): def parseConnections(self):
@ -354,7 +354,7 @@ class IMDb:
def getReleaseinfo(self, forcereload = False): def getReleaseinfo(self, forcereload = False):
if forcereload or not self.releaseinfoSource: if forcereload or not self.releaseinfoSource:
self.releaseinfoSource = read_url(self.releaseinfoUrl) self.releaseinfoSource = read_url_utf8(self.releaseinfoUrl)
return self.releaseinfoSource return self.releaseinfoSource
def parseReleaseinfo(self): def parseReleaseinfo(self):
@ -373,7 +373,7 @@ class IMDb:
def getBusiness(self, forcereload = False): def getBusiness(self, forcereload = False):
if forcereload or not self.businessSource: if forcereload or not self.businessSource:
self.businessSource = read_url(self.businessUrl) self.businessSource = read_url_utf8(self.businessUrl)
return self.businessSource return self.businessSource
def parseBusiness(self): def parseBusiness(self):