pandora/pandora/archive/views.py

758 lines
26 KiB
Python

# -*- coding: utf-8 -*-
from __future__ import division, print_function, absolute_import
import os.path
from datetime import datetime
from django.shortcuts import get_object_or_404, redirect, render
from django.conf import settings
from django.db.models import Count, Q
from six import string_types
from celery.utils import get_full_cls_name
from celery.backends import default_backend
import ox
from oxdjango.decorators import login_required_json
from oxdjango.shortcuts import render_to_json_response, get_object_or_404_json, json_response
from item import utils
from item.models import get_item, Item
from item.views import parse_query
import item.tasks
from oxdjango.api import actions
from changelog.models import add_changelog
from taskqueue.models import Task
from . import models
from . import queue
from . import tasks
from . import external
from .chunk import process_chunk
@login_required_json
def removeVolume(request, data):
'''
Removes a volume
takes {} // undocumented
returns {} // undocumented
'''
user = request.user
try:
volume = models.Volume.objects.get(user=user, name=data['volume'])
volume.files.delete()
volume.delete()
response = json_response()
except models.Volume.DoesNotExist:
response = json_response(status=404, text='volume not found')
return render_to_json_response(response)
actions.register(removeVolume, cache=False)
@login_required_json
def update(request, data):
'''
Undocumented
2 steps:
send files
{volume: 'Videos', files: [{oshash:, path:, mtime:, ,,}]}
send info about changed/new files
{volume: 'Videos', info: {oshash: {...}]}
call volume/files first and fill in requested info after that
takes {
volume: '',
files: [
{oshash:, path:, mtime:, },
...
],
info: {oshash: object}
}
returns {
info: list, // list of files that need info
data: list, // list of flies that should be encoded to highest profile and uploaded
file: list // list of files that should be uploaded as is
}
'''
user = request.user
upload_only = data.get('upload', False)
response = json_response({'info': [], 'data': [], 'file': []})
volume = None
if 'files' in data:
t = tasks.update_files.delay(user.username, data['volume'], data['files'])
response['data']['taskId'] = t.task_id
user_profile = user.profile
user_profile.files_updated = datetime.now()
user_profile.save()
if 'info' in data and data['info']:
t = tasks.update_info.delay(user.username, data['info'])
response['data']['taskId'] = t.task_id
if not upload_only:
all_files = models.Instance.objects.filter(volume__user=user)
files = all_files.filter(file__available=False)
if volume:
files = files.filter(volume=volume)
response['data']['info'] = [f.file.oshash for f in all_files.filter(Q(file__info='{}') | Q(file__size=0))]
response['data']['data'] = [f.file.oshash for f in files.filter(file__is_video=True,
file__available=False,
file__wanted=True)]
response['data']['data'] += [f.file.oshash for f in files.filter(file__is_audio=True,
file__available=False,
file__wanted=True)]
if list(filter(lambda l: l['id'] == 'subtitles', settings.CONFIG['layers'])):
qs = files.filter(
file__is_subtitle=True,
file__available=False
).filter(
Q(path__endswith='.srt') | Q(path__endswith='.vtt')
)
response['data']['file'] = [f.file.oshash for f in qs]
else:
response['data']['file'] = []
return render_to_json_response(response)
actions.register(update, cache=False)
@login_required_json
def upload(request, data=None):
'''
Uploads one or more media files for a given item
takes {
id: string // item id
frame: [] // one or more frames
file: [] // one or more files
}
returns {
info: object, // undocumented
rename: object // undocumented
}
see: add, edit, find, get, lookup, remove
'''
response = json_response({})
f = get_object_or_404_json(models.File, oshash=request.POST['id'])
if 'frame' in request.FILES:
if f.editable(request.user):
f.frames.all().delete()
for frame in request.FILES.getlist('frame'):
name = frame.name
# float required?
position = float(os.path.splitext(name)[0])
fr, created = models.Frame.objects.get_or_create(file=f, position=position)
if fr.frame:
fr.frame.delete()
fr.frame.save(name, frame)
os.chmod(fr.frame.path, 0o644)
fr.save()
f.item.select_frame()
f.item.save()
item.tasks.update_poster.delay(f.item.public_id)
if 'file' in request.FILES:
if not f.available:
if f.data:
f.data.delete()
f.data.save('data.raw', request.FILES['file'])
f.save()
os.chmod(f.data.path, 0o644)
f.item.update_selected()
if not f.item.rendered:
item.tasks.update_timeline.delay(f.item.public_id)
else:
item.tasks.load_subtitles.delay(f.item.public_id)
response = json_response(text='file saved')
else:
response = json_response(status=403, text='permission denied')
return render_to_json_response(response)
actions.register(upload, cache=False)
@login_required_json
def addMedia(request, data):
'''
Adds media files to a given item
takes {
filename: string, // filename
id: string, // oshash of the file
info: {}, // undocumented
item: string // item id
}
returns {
item: id // item id
}
see: editMedia, findMedia, moveMedia, removeMedia
'''
response = json_response({})
oshash = data.pop('id')
if not request.user.profile.capability('canAddItems'):
response = json_response(status=403, text='permission denied')
elif models.File.objects.filter(oshash=oshash).count() > 0:
f = models.File.objects.get(oshash=oshash)
if f.available:
response['status']['text'] = 'file exists'
response['data']['item'] = f.item.public_id
response['data']['itemUrl'] = request.build_absolute_uri('/%s' % f.item.public_id)
if not f.available:
add_changelog(request, data, f.item.public_id)
else:
if 'item' in data:
i = Item.objects.get(public_id=data['item'])
else:
title = ox.parse_movie_path(os.path.splitext(data['filename'])[0])['title']
i = Item()
i.data = {
'title': title,
'director': data.get('director', []),
}
i.user = request.user
i.save()
i.make_poster()
f = models.File(oshash=oshash, item=i)
f.path = data.get('filename', 'Untitled')
extension = f.path.split('.')
if len(extension) > 1:
extension = extension[-1].lower()
else:
extension = 'webm'
f.selected = True
if 'info' in data and data['info'] and isinstance(data['info'], dict):
f.info = data['info']
f.info['extension'] = extension
f.parse_info()
f.save()
response['data']['item'] = i.public_id
response['data']['itemUrl'] = request.build_absolute_uri('/%s' % i.public_id)
add_changelog(request, data, i.public_id)
return render_to_json_response(response)
actions.register(addMedia, cache=False)
@login_required_json
def firefogg_upload(request):
if not 'profile' in request.GET or not 'id' in request.GET:
return render(request, 'api.html', {
'api': [],
'settings': settings,
'sitename': settings.SITENAME
})
profile = request.GET['profile']
oshash = request.GET['id']
config = settings.CONFIG['video']
resolution, format = profile.split('p.')
resolution = int(resolution)
if resolution not in config['resolutions'] \
or format not in config['formats']:
response = json_response(status=500, text='invalid profile')
return render_to_json_response(response)
# handle video upload
if request.method == 'POST':
# post next chunk
if 'chunk' in request.FILES and oshash:
f = get_object_or_404(models.File, oshash=oshash)
if f.editable(request.user):
def save_chunk(chunk, offset, done):
return f.save_chunk_stream(chunk, offset, resolution, format, done)
response = process_chunk(request, save_chunk)
response['resultUrl'] = request.build_absolute_uri('/%s' % f.item.public_id)
if response.get('done'):
f.uploading = False
if response['result'] == 1:
f.queued = True
f.wanted = False
else:
f.queued = False
f.wanted = True
f.save()
try:
t = f.process_stream()
response['resultUrl'] = t.task_id
except:
pass
return render_to_json_response(response)
# init upload
elif oshash:
# 404 if oshash is not know, files must be registered via update api first
f = get_object_or_404(models.File, oshash=oshash)
if f.editable(request.user):
f.streams.all().delete()
f.delete_frames()
f.uploading = True
f.failed = False
f.save()
if f.item.rendered and f.selected:
Item.objects.filter(id=f.item.id).update(rendered=False)
Task.start(f.item, request.user)
add_changelog({
'user': request.user,
'action': 'upload',
}, {
'profile': profile,
'item': f.item.public_id,
}, f.oshash)
response = {
'uploadUrl': '/api/upload/?id=%s&profile=%s' % (f.oshash, profile),
'url': request.build_absolute_uri('/%s' % f.item.public_id),
'result': 1
}
return render_to_json_response(response)
else:
response = json_response(status=404, text='permission denied')
response = json_response(status=400, text='this request requires POST')
return render_to_json_response(response)
@login_required_json
def direct_upload(request):
if 'id' in request.GET:
oshash = request.GET['id']
else:
oshash = request.POST['id']
response = json_response(status=400, text='this request requires POST')
if 'chunk' in request.FILES:
file = models.File.objects.get(oshash=oshash)
if file.editable(request.user):
response = process_chunk(request, file.save_chunk)
response['resultUrl'] = request.build_absolute_uri(file.item.get_absolute_url())
if response.get('done'):
file.uploading = False
if response['result'] == 1:
file.queued = True
file.wanted = False
else:
file.queued = False
file.wanted = True
file.save()
# try/execpt so it does not fail if rabitmq is down
try:
t = file.extract_stream()
response['resultUrl'] = t.task_id
except:
print('ERROR: rabbitmq is down')
pass
return render_to_json_response(response)
# init upload
else:
file, created = models.File.objects.get_or_create(oshash=oshash)
if file.editable(request.user):
if created:
if not file.item:
file.item = get_item({
'title': oshash
}, user=request.user)
else: # remove previous uploads
file.streams.all().delete()
file.delete_frames()
if file.item.rendered and file.selected:
Item.objects.filter(id=file.item.id).update(rendered=False)
file.uploading = True
file.save()
Task.start(file.item, request.user)
add_changelog({
'user': request.user,
'action': 'upload',
}, {
'item': file.item.public_id,
}, file.oshash)
upload_url = '/api/upload/direct/?id=%s' % file.oshash
return render_to_json_response({
'uploadUrl': upload_url,
'url': request.build_absolute_uri(file.item.get_absolute_url()),
'result': 1
})
else:
response = json_response(status=403, text='permission denied')
return render_to_json_response(response)
@login_required_json
def getTaskStatus(request, data):
'''
Gets the status for a given task
takes {
id: string // taskId
}
returns {
id: string, // taskId
status: string, // status, 'PENDING' or 'OK'
result: object // result data
}
notes: To be deprecated, will be wrapped in regular API call.
'''
# FIXME: should check if user has permissions to get status
if 'id' in data:
task_id = data['id']
elif 'taskId' in data:
task_id = data['taskId']
else:
task_id = data['task_id']
response = json_response(status=200, text='ok')
status = default_backend.get_status(task_id)
res = default_backend.get_result(task_id)
response['data'] = {
'id': task_id,
'status': status
}
if isinstance(res, dict):
response['data'].update(res)
else:
response['data']['result'] = res
if status in default_backend.EXCEPTION_STATES:
traceback = default_backend.get_traceback(task_id)
response['data'].update({
'result': str(res),
'exc': get_full_cls_name(res.__class__),
'traceback': traceback
})
return render_to_json_response(response)
actions.register(getTaskStatus, cache=False)
actions.register(getTaskStatus, action='taskStatus', cache=False)
@login_required_json
def moveMedia(request, data):
'''
Moves one or more media files from one item to another
takes {
ids: [string], // list of file ids
item: id // target item id
}
returns {}
notes: This will *not* (yet) shift the corresponding annotations.
see: addMedia, editMedia, findMedia, removeMedia
'''
response = json_response()
t = tasks.move_media.delay(data, request.user.username)
response['data']['taskId'] = t.task_id
return render_to_json_response(response)
actions.register(moveMedia, cache=False)
@login_required_json
def editMedia(request, data):
'''
Edits data for one or more media files
takes {
files: [
{
id: string, // file id
key: value, // property id and new value
... more key/value pairs
},
// more media files
]
}
returns {}
notes: Possible keys are 'episodes', 'extension', 'ignore', 'language',
'part', 'partTitle' and 'version'.
see: addMedia, findMedia, moveMedia, removeMedia
'''
ignore = []
dont_ignore = []
save_items = set()
update_timeline = set()
response = json_response(status=200, text='updated')
response['data']['files'] = []
for info in data['files']:
f = get_object_or_404_json(models.File, oshash=info['id'])
if f.editable(request.user):
if 'ignore' in info:
if info['ignore']:
ignore.append(info['id'])
else:
dont_ignore.append(info['id'])
update = False
for key in f.PATH_INFO:
if key in info:
f.info[key] = info[key]
if key == 'language' and (f.is_video or f.is_audio):
save_items.add(f.item.id)
if key == 'part' and (f.is_video or f.is_audio):
if f.item.rendered:
f.item.rendered = False
f.item.save()
Task.start(f.item, request.user)
update_timeline.add(f.item.id)
update = True
if update:
f.save()
response['data']['files'].append(f.json(user=request.user))
else:
response['data']['files'].append({'id': info['id'], 'error': 'permission denied'})
if ignore:
models.Instance.objects.filter(file__oshash__in=ignore).update(ignore=True)
if dont_ignore:
models.Instance.objects.filter(file__oshash__in=dont_ignore).update(ignore=False)
if ignore or dont_ignore:
files = models.File.objects.filter(oshash__in=ignore+dont_ignore)
ids = []
for i in Item.objects.filter(files__in=files).distinct():
i.update_selected()
i.update_wanted()
ids.append(i.public_id)
if ignore:
qs = models.File.objects.filter(oshash__in=ignore, instances__id=None, selected=True)
if qs.count():
ids += [f.item.public_id for f in qs]
qs.update(selected=False)
if dont_ignore:
qs = models.File.objects.filter(oshash__in=dont_ignore, instances__id=None, selected=False)
if qs.count():
ids += [f.item.public_id for f in qs]
qs.update(selected=True)
for id in list(set(ids)):
item.tasks.update_timeline.delay(id)
if save_items | update_timeline:
for i in Item.objects.filter(id__in=list(save_items | update_timeline)):
if i.id in save_items:
i.save()
if i.id in update_timeline:
item.tasks.update_timeline.delay(i.public_id)
add_changelog(request, data, [f['id'] for f in response['data']['files']])
return render_to_json_response(response)
actions.register(editMedia, cache=False)
@login_required_json
def removeMedia(request, data):
'''
Removes one or more media files from a given item
takes {} // undocumented
returns {} // undocumented
see: addMedia, editMedia, findMedia, moveMedia
'''
response = json_response()
if request.user.profile.get_level() == 'admin':
qs = models.File.objects.filter(oshash__in=data['ids'], instances__id=None)
selected = set([f.item.id for f in qs if f.selected])
items = list(set([f.item.id for f in qs]))
qs.delete()
for i in Item.objects.filter(id__in=items):
if i.id in selected:
i.update_timeline()
else:
i.save()
add_changelog(request, data, data['ids'])
else:
response = json_response(status=403, text='permission denied')
return render_to_json_response(response)
actions.register(removeMedia, cache=False)
def getPath(request, data):
'''
Undocumented
change file / item link
takes {
id: [hash of file]
}
returns {
id: path
}
'''
response = json_response()
ids = data['id']
if isinstance(ids, string_types):
ids = [ids]
for f in models.File.objects.filter(oshash__in=ids).values('path', 'oshash').order_by('sort_path'):
response['data'][f['oshash']] = f['path']
return render_to_json_response(response)
actions.register(getPath, cache=True)
def lookup_file(request, oshash):
oshash = oshash.replace('/', '')
f = get_object_or_404(models.File, oshash=oshash)
return redirect('%s/media' % f.item.get_absolute_url())
def _order_query(qs, sort, prefix=''):
order_by = []
if len(sort) == 1:
sort.append({'operator': '+', 'key': 'path'})
sort.append({'operator': '-', 'key': 'created'})
for e in sort:
operator = e['operator']
if operator != '-':
operator = ''
key = {
'id': 'item__public_id',
'users': 'instances__volume__user__username',
'resolution': 'width',
'path': 'sort_path',
'partTitle': 'part_title',
}.get(e['key'], e['key'])
#if operator=='-' and '%s_desc'%key in models.ItemSort.descending_fields:
# key = '%s_desc' % key
order = '%s%s%s' % (operator, prefix, key)
order_by.append(order)
if key == 'part':
order = '%s%s%s' % (operator, prefix, 'sort_path')
order_by.append(order)
if order_by:
qs = qs.order_by(*order_by)
return qs
def parse_query(data, user):
query = {}
query['range'] = [0, 100]
query['sort'] = [{'key': 'path', 'operator': '+'}]
for key in ('sort', 'keys', 'group', 'range', 'position', 'positions'):
if key in data:
query[key] = data[key]
if [r for r in query['range'] if not isinstance(r, int)]:
query['range'] = [0, 0]
query['qs'] = models.File.objects.find(data, user)
return query
def findMedia(request, data):
'''
Finds media files
takes {
query: object, // query object, see `find`
sort: array, // list of sort objects, see `find`
range: [int, int] // range of results to return
}
returns {
items: [object] // list of items
}
see: addMedia, editMedia, find, moveMedia, removeMedia
'''
if not data.get('sort'):
data['sort'] = [{'key': 'path', 'operator': '+'}]
query = parse_query(data, request.user)
response = json_response({})
if 'group' in query:
print('findMedia does not support group query')
elif 'positions' in query:
qs = _order_query(query['qs'], query['sort'])
response['data']['positions'] = {}
ids = list(qs.values_list('oshash', flat=True))
response['data']['positions'] = utils.get_positions(ids, query['positions'])
elif 'keys' in query:
response['data']['items'] = []
qs = _order_query(query['qs'], query['sort'])
qs = qs.select_related()
keys = query['keys']
qs = qs[query['range'][0]:query['range'][1]]
response['data']['items'] = [f.json(keys, user=request.user) for f in qs]
else: # otherwise stats
files = query['qs']
response['data']['items'] = files.count()
return render_to_json_response(response)
actions.register(findMedia)
def parsePath(request, data): # parse path and return info
'''
Parses a path
takes {
path: string // undocumented
}
returns {
imdb: string // undocumented
}
'''
path = data['path']
response = json_response(ox.parse_movie_path(path))
return render_to_json_response(response)
actions.register(parsePath)
def getMediaInfo(request, data):
'''
Gets media file info, undocumented
takes {
id: string // oshash of media file
}
returns {
item: string, // item id
file: string // oshash of source file
resolution: int // stream resolution
index: int // stream index
}
'''
f = None
resolution = None
index = None
qs = models.Stream.objects.filter(oshash=data['id'])
if qs.count() > 0:
s = qs[0]
f = s.file
resolution = s.resolution
index = s.get_index()
else:
qs = models.File.objects.filter(oshash=data['id'])
if qs.count() > 0:
f = qs[0]
response = json_response()
if f:
response['data'] = {
'file': f.oshash,
'item': f.item.public_id
}
if resolution:
response['data']['resolution'] = resolution
if index:
response['data']['index'] = index
return render_to_json_response(response)
actions.register(getMediaInfo)
def getEncodingStatus(request, data):
response = json_response()
response['data']['status'] = queue.status()
return render_to_json_response(response)
actions.register(getEncodingStatus, cache=False)
@login_required_json
def getMediaUrlInfo(request, data):
'''
Get info (title, duration,...) about given media url,
if url is a playlist, result has info about each item.
takes {
url: string // url
}
returns {
items: [{title, url,...}] // info for each url found
}
'''
if not request.user.profile.capability('canAddItems'):
response = json_response(status=403, text='permission denied')
else:
response = json_response()
response['data']['items'] = external.get_info(data['url'])
return render_to_json_response(response)
actions.register(getMediaUrlInfo, cache=True)
@login_required_json
def addMediaUrl(request, data):
'''
Import video from url and add to item
takes {
url: string, // url
item: string // item
}
returns {
taskId: string, // taskId
}
'''
if not request.user.profile.capability('canAddItems'):
response = json_response(status=403, text='permission denied')
else:
response = json_response()
i = Item.objects.get(public_id=data['item'])
Task.start(i, request.user)
t = tasks.download_media.delay(data['item'], data['url'])
response['data']['taskId'] = t.task_id
add_changelog(request, data, data['item'])
return render_to_json_response(response)
actions.register(addMediaUrl, cache=False)