add import scripts
This commit is contained in:
parent
925136e79b
commit
c5cd5cb8cb
2 changed files with 189 additions and 0 deletions
2
download.sh
Executable file
2
download.sh
Executable file
|
|
@ -0,0 +1,2 @@
|
||||||
|
wget -r -c -np https://www.indymedia.org.uk/media/
|
||||||
|
wget -r -c -np https://www.indymedia.org.uk/content/
|
||||||
187
video_import.py
Normal file
187
video_import.py
Normal file
|
|
@ -0,0 +1,187 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
from datetime import datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import ox
|
||||||
|
import pandora_client
|
||||||
|
import lxml.html
|
||||||
|
|
||||||
|
def get_html(doc, classname):
|
||||||
|
element = '*'
|
||||||
|
d = doc.xpath("//%s[contains(@class, '%s')]" % (element, classname))
|
||||||
|
if not len(d):
|
||||||
|
return ''
|
||||||
|
d = d[0]
|
||||||
|
html = lxml.html.tostring(d, pretty_print=True)
|
||||||
|
html = ox.sanitize_html(html).replace('<a></a>', '').strip()
|
||||||
|
while html.endswith('<br><br>'):
|
||||||
|
html = html.rstrip('<br><br>').strip()
|
||||||
|
html = '\n'.join(html.split('\n')[1:-1]).strip()
|
||||||
|
html = re.sub('<img src="/img/.*?">', '', html)
|
||||||
|
html = html.replace('href="/', 'href="https://www.indymedia.org.uk/')
|
||||||
|
html = html.replace('src="/', 'src="https://www.indymedia.org.uk/')
|
||||||
|
return html
|
||||||
|
|
||||||
|
def parse_content(path):
|
||||||
|
info = {}
|
||||||
|
with open(path) as fd:
|
||||||
|
data = fd.read().decode('utf-8')
|
||||||
|
doc = lxml.html.fromstring(data)
|
||||||
|
title = doc.xpath("//a[contains(@class, 'arttitle')]")[0].text_content().strip()
|
||||||
|
url = 'https://' + path[path.index('www.indymedia.org.uk'):].replace('/content/', '/en/')
|
||||||
|
info['links'] = [url]
|
||||||
|
info['title'] = title
|
||||||
|
p = doc.xpath("//p[contains(@class, 'date')]")[0]
|
||||||
|
date = p.text_content().strip()
|
||||||
|
date = re.compile('\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}').findall(date)
|
||||||
|
if date:
|
||||||
|
info['date'] = datetime.strptime(date[0], '%d.%m.%Y %H:%M').strftime('%Y-%m-%d %H:%M')
|
||||||
|
info['themes'] = [a.text_content() for a in p.xpath('//a') if a.attrib.get('href', '').startswith('/en/topics')]
|
||||||
|
info['themes'] += [a.text_content() for a in p.xpath('//a') if a.attrib.get('href', '').startswith('/en/actions')]
|
||||||
|
l = ', '.join([a.text_content() for a in p.xpath('//a') if a.attrib.get('href', '').startswith('/en/regions')])
|
||||||
|
if l:
|
||||||
|
info['location'] = l
|
||||||
|
|
||||||
|
mids = set()
|
||||||
|
media = []
|
||||||
|
for id, title in re.compile('href="(/media/.*?)">(.*?)</a>', re.DOTALL).findall(data) \
|
||||||
|
+ re.compile('href="http://www.indymedia.org.uk(/media/.*?)">(.*?)</a>', re.DOTALL).findall(data):
|
||||||
|
id = ox.decode_html(re.sub('\s+', ' ', id.strip()).replace('//', '/'))
|
||||||
|
title = ox.decode_html(re.sub('\s+', ' ', title.strip()).replace('//', '/'))
|
||||||
|
if id not in mids:
|
||||||
|
mids.add(id)
|
||||||
|
media.append([id, title])
|
||||||
|
|
||||||
|
info['media'] = media
|
||||||
|
|
||||||
|
info['video'] = [m for m in media if m[0].split('.')[-1] in ox.file.EXTENSIONS['video']]
|
||||||
|
|
||||||
|
images = re.compile('src="(/images/.*?)".*?alt="(.*?)">', re.DOTALL).findall(data)
|
||||||
|
images = [[ox.decode_html(re.sub('\s+', ' ', e.strip()).replace('//', '/')) for e in r] for r in images]
|
||||||
|
info['images'] = images
|
||||||
|
|
||||||
|
info['abstract'] = get_html(doc, 'intro')
|
||||||
|
info['content'] = get_html(doc, 'articlecontent')
|
||||||
|
try:
|
||||||
|
d = doc.xpath("//p[contains(@class, 'creator')]")[0]
|
||||||
|
info['creator'] = {
|
||||||
|
'name': d.xpath('.//strong')[0].text_content()
|
||||||
|
}
|
||||||
|
info['depositor'] = info['creator']['name']
|
||||||
|
info['creator']['url'] = d.xpath('.//a')[0].attrib['href']
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
content = []
|
||||||
|
content.append(info['abstract'])
|
||||||
|
images = []
|
||||||
|
for image in info['images']:
|
||||||
|
img = u'<figure><img src="{url}"><figcaption>{info}</figcaption></figure>'.format(
|
||||||
|
url='https://www.indymedia.org.uk' + image[0], info=image[1])
|
||||||
|
images.append(img)
|
||||||
|
content.append(u'\n'.join(images))
|
||||||
|
content.append(info['content'])
|
||||||
|
if 'creator' in info and 'url' in info['creator']:
|
||||||
|
content.append(u'Creator: <a href="{url}">{name}</a>'.format(**info['creator']))
|
||||||
|
info['summary'] = u'\n\n'.join(content).strip()
|
||||||
|
info['summary'] = re.sub(info['summary'], '\n\n+', '\n\n')
|
||||||
|
return info
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
api_url = 'https://urg.0x2620.org/api/'
|
||||||
|
api = pandora_client.API(api_url)
|
||||||
|
api.signin(username='import', password='indyport')
|
||||||
|
|
||||||
|
failed = []
|
||||||
|
missing = []
|
||||||
|
invalid = []
|
||||||
|
fileids = set()
|
||||||
|
for root, folders, files in os.walk('www.indymedia.org.uk/content/', topdown=True):
|
||||||
|
folders.sort()
|
||||||
|
for f in sorted(files):
|
||||||
|
if f.endswith('.html'):
|
||||||
|
id = f.split('/')[-1].split('.')[0]
|
||||||
|
url = os.path.join(root, f).replace('www.indymedia.org.uk/content/', '')
|
||||||
|
path = os.path.join(root, f)
|
||||||
|
url = 'https://www.indymedia.org.uk/en/' + url
|
||||||
|
with open(path) as fd:
|
||||||
|
data = fd.read()
|
||||||
|
if '<!-- content -->' in data:
|
||||||
|
try:
|
||||||
|
p = parse_content(path)
|
||||||
|
except:
|
||||||
|
if '/media/' in data:
|
||||||
|
failed.append(url)
|
||||||
|
continue
|
||||||
|
if p['video']:
|
||||||
|
print(url)
|
||||||
|
r = api.find({
|
||||||
|
'query': {'conditions': [{'key': 'links', 'value': url, 'operator': '=='}]},
|
||||||
|
'keys': ['id'],
|
||||||
|
'range': [0, 100]
|
||||||
|
})
|
||||||
|
info = {}
|
||||||
|
for key in ('title', 'summary', 'links', 'themes', 'location', 'date', 'depositor'):
|
||||||
|
if key in p:
|
||||||
|
info[key] = p[key]
|
||||||
|
if r['data']['items']:
|
||||||
|
item_id = r['data']['items'][0]['id']
|
||||||
|
'''
|
||||||
|
info['id'] = item_id
|
||||||
|
r = api.edit(**info)
|
||||||
|
if r['status']['code'] != 200:
|
||||||
|
print(r)
|
||||||
|
assert(r['status']['code'] == 200)
|
||||||
|
# fixme, sync videos
|
||||||
|
'''
|
||||||
|
else:
|
||||||
|
item_id = api.add(title=p['title'])['data']['id']
|
||||||
|
info['id'] = item_id
|
||||||
|
r = api.edit(**info)
|
||||||
|
if r['status']['code'] != 200:
|
||||||
|
print(item_id)
|
||||||
|
print(info)
|
||||||
|
print(r)
|
||||||
|
assert(r['status']['code'] == 200)
|
||||||
|
for id, title in p['video']:
|
||||||
|
f = 'www.indymedia.org.uk' + id
|
||||||
|
f = os.path.abspath(f)
|
||||||
|
if not os.path.exists(f):
|
||||||
|
missing.append(f)
|
||||||
|
continue
|
||||||
|
info = ox.avinfo(f)
|
||||||
|
if 'error' in info or 'oshash' not in info:
|
||||||
|
invalid.append(f)
|
||||||
|
continue
|
||||||
|
oshash = info['oshash']
|
||||||
|
if oshash in fileids:
|
||||||
|
print('WTF', oshash, 'known')
|
||||||
|
continue
|
||||||
|
fileids.add(oshash)
|
||||||
|
if 'path' in info:
|
||||||
|
del info['path']
|
||||||
|
# print('adding', f, info)
|
||||||
|
r = api.addMedia({
|
||||||
|
'id': oshash,
|
||||||
|
'item': item_id,
|
||||||
|
'filename': id
|
||||||
|
})
|
||||||
|
print(r)
|
||||||
|
assert(r['status']['code'] == 200)
|
||||||
|
# upload media file
|
||||||
|
url = '%supload/direct/' % api_url
|
||||||
|
r = api.upload_chunks(url, f, {
|
||||||
|
'id': oshash
|
||||||
|
})
|
||||||
|
print('chunk upload', oshash, r)
|
||||||
|
assert(r)
|
||||||
|
|
||||||
|
with open('import_failed.json', 'w') as fd:
|
||||||
|
json.dump(failed, fd, indent=4)
|
||||||
|
with open('import_missing.json', 'w') as fd:
|
||||||
|
json.dump(missing, fd, indent=4)
|
||||||
|
with open('import_invalid.json', 'w') as fd:
|
||||||
|
json.dump(invalid, fd, indent=4)
|
||||||
Loading…
Reference in a new issue