154 lines
No EOL
6 KiB
Python
154 lines
No EOL
6 KiB
Python
from datetime import datetime
|
|
import re
|
|
|
|
from BeautifulSoup import BeautifulSoup
|
|
|
|
from oxutils.cache import getUrl
|
|
from oxutils.html import stripTags
|
|
import oxutils.net
|
|
|
|
def output(news):
|
|
for new in news:
|
|
print '\n'.join([new['section'] + ', ' + new['dateString'], new['title0'] + ': ' + new['title1'], new['description'], ''])
|
|
|
|
def getNews(year, month, day):
|
|
sections = [
|
|
'politik', 'wirtschaft', 'panorama', 'sport', 'kultur', 'netzwelt',
|
|
'wissenschaft', 'unispiegel', 'schulspiegel', 'reise', 'auto'
|
|
]
|
|
dt = datetime(year, month, day)
|
|
day = int(dt.strftime('%j'))
|
|
date = dt.strftime('%d.%m.%Y')
|
|
news = []
|
|
for section in sections:
|
|
url = 'http://www.spiegel.de/%s/0,1518,archiv-%d-%03d,00.html' % (section, year, day)
|
|
print url
|
|
html = getUrl(url)
|
|
for item in re.compile('<div class="spTeaserCenterpage(.*?)</p>', re.DOTALL).findall(html):
|
|
dateString = stripTags(re.compile('<div class="spDateTime">(.*?)</div>', re.DOTALL).findall(item)[0]).strip()
|
|
try:
|
|
description = re.compile('<p>(.*?)<', re.DOTALL).findall(item)[0].strip()
|
|
except:
|
|
description = ''
|
|
try:
|
|
imageUrl = re.compile('<img src="(.*?)"').findall(item)[0]
|
|
except:
|
|
imageUrl = ''
|
|
if dateString[:10] == date and description and imageUrl:
|
|
# print item
|
|
new = {}
|
|
if len(dateString) == 10:
|
|
new['date'] = '%s-%s-%s 00:00' % (dateString[6:10], dateString[3:5], dateString[:2])
|
|
else:
|
|
new['date'] = '%s-%s-%s %s:%s' % (dateString[6:10], dateString[3:5], dateString[:2], dateString[12:14], dateString[15:17])
|
|
new['dateString'] = dateString
|
|
new['description'] = formatString(description)
|
|
new['imageUrl'] = imageUrl
|
|
new['section'] = formatSection(section)
|
|
new['title'] = formatString(re.compile('title=[\'|"](.*?)[\'|"] />', re.DOTALL).findall(item)[0])
|
|
new['title0'] = re.compile('(.*?): ').findall(new['title'])[0]
|
|
new['title1'] = re.compile(': (.*?)$').findall(new['title'])[0]
|
|
new['url'] = 'http://www.spiegel.de' + re.compile('<a href="(.*?)"').findall(item)[0]
|
|
news.append(new)
|
|
print dateString + ' - ok'
|
|
elif not description:
|
|
print dateString + ' - no description'
|
|
elif not imageUrl:
|
|
print dateString + ' - no image'
|
|
return news
|
|
|
|
def formatString(string):
|
|
return string.replace('\n', ' ').replace(' ', ' ').strip()
|
|
|
|
def formatSection(string):
|
|
return string[:1].upper() + string[1:].replace('spiegel', 'SPIEGEL')
|
|
|
|
def formatSubsection(string):
|
|
subsection = {
|
|
'abi': 'Abi - und dann?',
|
|
'formel1': 'Formel 1',
|
|
'jobundberuf': 'Job & Beruf',
|
|
'leben': 'Leben U21',
|
|
'mensch': 'Mensch & Technik',
|
|
'sonst': '',
|
|
'staedte': u'St\xc3dte',
|
|
'ussports': 'US-Sports',
|
|
'wunderbar': 'wunderBAR'
|
|
}
|
|
if subsection.has_key(string):
|
|
return subsection[string]
|
|
return string[:1].upper() + string[1:]
|
|
|
|
|
|
class Spiegel:
|
|
def __init__(self, year, week):
|
|
self.year = year
|
|
self.week = week
|
|
self.coverUrl = 'http://www.spiegel.de/static/epaper/SP/%d/%d/ROSPANZ%d0%02d0001-312.jpg' % (self.year, self.week, self.year, self.week)
|
|
if not oxutils.net.exists(self.coverUrl):
|
|
self.coverUrl = ''
|
|
return
|
|
self.contentsUrl = 'http://service.spiegel.de/digas/servlet/epaper?Q=SP&JG=%d&AG=%d&SE=1&AN=INHALT' % (self.year, self.week)
|
|
|
|
def getContents(self):
|
|
self.contents = []
|
|
soup = BeautifulSoup(getUrl(self.contentsUrl))
|
|
for item in soup('a', {'href': re.compile('http://service.spiegel.de/digas/servlet/epaper\?Q=SP&JG=')}):
|
|
item = str(item)
|
|
title = stripTags(item).strip()
|
|
page = int(re.compile('&SE=(.*?)"').findall(item)[0])
|
|
self.contents.append({'title': title, 'page': page})
|
|
return self.contents
|
|
|
|
def archiveNews():
|
|
'''
|
|
this is just an example of an archiving application
|
|
'''
|
|
import os
|
|
import simplejson
|
|
archivePath = '/Volumes/Rolux Home/Desktop/Data/spiegel.de'
|
|
for y in range(2007, 2008):
|
|
for m in range(1, 13):
|
|
for d in range(1, 32):
|
|
news = getNews(y, m ,d)
|
|
for new in news:
|
|
dirname = archivePath + '/' + new['date'][0:4] + '/' + new['date'][5:7] + new['date'][8:10] + '/' + new['date'][11:13] + new['date'][14:16]
|
|
if not os.path.exists(dirname):
|
|
os.makedirs(dirname)
|
|
filename = dirname + '/' + new['url'].split('/')[-1][:-5] + '.json'
|
|
data = simplejson.dumps(new, ensure_ascii = False)
|
|
f = open(filename, 'w')
|
|
f.write(data)
|
|
f.close()
|
|
filename = filename[:-4] + 'txt'
|
|
data = '\n'.join([new['title0'], new['title1'], new['description']])
|
|
f = open(filename, 'w')
|
|
f.write(data)
|
|
f.close()
|
|
filename = dirname + '/' + new['imageUrl'].split('/')[-1]
|
|
data = getUrl(new['imageUrl'])
|
|
f = open(filename, 'w')
|
|
f.write(data)
|
|
f.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# spiegel = Spiegel(2008, 8)
|
|
# print spiegel.getContents()
|
|
# news = News(2001, 9, 10)
|
|
# output(news.getNews())
|
|
'''
|
|
x = []
|
|
for d in range(10, 30):
|
|
print '2/%d' % d
|
|
news = getNews(2008, 2, d)
|
|
for new in news:
|
|
strings = new['url'].split('/')
|
|
string = formatSection(strings[3])
|
|
if len(strings) == 6:
|
|
string += '/' + formatSubsection(strings[4])
|
|
if not string in x:
|
|
x.append(string)
|
|
print x
|
|
'''
|
|
archiveNews() |