import urllib.parse from django.utils.timezone import datetime, timedelta from django.utils import timezone import json import requests import lxml.html from django.conf import settings from django.contrib.auth import get_user_model from django.db import models from django.db.models.functions import ExtractWeek, ExtractYear from django.urls import reverse from django.utils.timesince import timesince import ox User = get_user_model() class Settings(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) key = models.CharField(max_length=1024, unique=True) value = models.JSONField(default=dict, editable=False) class Item(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) title = models.CharField(max_length=1024) url = models.CharField(max_length=1024, unique=True) description = models.TextField(default="", blank=True, editable=False) published = models.DateTimeField(default=None, null=True, blank=True) announced = models.DateTimeField(null=True, default=None, blank=True, editable=False) data = models.JSONField(default=dict, editable=False) user = models.ForeignKey(User, null=True, related_name='items', on_delete=models.CASCADE) def save(self, *args, **kwargs): #if self.url and not self.data.get("url") == self.url: if self.url: self.update_data() if self.use_hue: if "hue" in self.data: del self.data["hue"] self.get_hue() super().save(*args, **kwargs) def __str__(self): return '%s (%s)' % (self.title, self.url) def public_comments(self): return self.comments.exclude(published=None) def public_comments_json(self): comments = [] for comment in self.public_comments(): comments.append({ "name": comment.name, "date": comment.date, "text": comment.text, }) return json.dumps(comments) @classmethod def all_public(cls, now=None): if now is None: now = timezone.make_aware(datetime.now(), timezone.get_default_timezone()) qs = cls.objects.exclude(published=None).filter(published__lte=now).order_by('-published') archive = qs.annotate(year=ExtractYear('published')).annotate(week=ExtractWeek('published')) return archive @classmethod def public(cls, now=None): if now is None: now = timezone.make_aware(datetime.now(), timezone.get_default_timezone()) qs = cls.all_public(now) cal = now.date().isocalendar() monday = now.date() - timedelta(days=now.date().isocalendar().weekday - 1) monday = timezone.datetime(monday.year, monday.month, monday.day, tzinfo=now.tzinfo) current_week = Week.objects.filter(monday=monday).first() if current_week and current_week.is_break: return current_week, qs first_post = qs.filter(published__gt=monday).first() if first_post and first_post.published < now: week = qs.filter(published__gt=monday) elif not first_post: while qs.exists() and not first_post: monday = monday - timedelta(days=7) current_week = Week.objects.filter(monday=monday).first() if current_week and current_week.is_break: return current_week, qs first_post = qs.filter(published__gt=monday).first() week = qs.filter(published__gt=monday) else: last_monday = monday - timedelta(days=7) week = qs.filter(published__gt=last_monday) archive = qs.exclude(id__in=week) return week, archive def get_week(self): return int(self.published.strftime('%W')) def get_year(self): return int(self.published.strftime('%Y')) def get_monday(self): d = '%s-W%s' % (self.get_year(), self.get_week()) return datetime.strptime(d + '-1', "%Y-W%W-%w").strftime('%Y-%m-%d') @property def use_hue(self): monday = self.get_monday() week = Week.objects.filter(monday=monday).first() if week: return week.use_hue return False def get_absolute_url(self): return reverse('item', kwargs={'id': self.id}) def full_url(self): return settings.URL + self.get_absolute_url() def update_data(self): self.data.update(self.parse_url()) if "hue" in self.data: del self.data["hue"] def parse_url(self): content = requests.get(self.url).text doc = lxml.html.fromstring(content) data = {} for meta in doc.cssselect('meta'): key = meta.attrib.get('name') if not key: key = meta.attrib.get('property') value = meta.attrib.get('content') if key and value: if key in ('viewport', ): continue key = key.replace('og:', '') data[key] = value data["url"] = self.url if "m/documents" in self.url: data["type"] = "document" data["thumbnail"] = data["thumbnail"].replace('/512p', '/1024p') return data def get_hue(self, update=False): if "hue" in self.data: return self.data["hue"] if update: self.save() return self.data.get("hue") hue = None parts = self.url.split('/') url = '/'.join(parts[:3]) + '/api/' if parts[4] == 'edits': edit = urllib.parse.unquote(parts[5]).replace('_', ' ') request = { "action": "getEdit", "data": { "id": edit, "keys": [] } } response = requests.post(url, json=request).json() clips = response["data"]["clips"] if clips: hue = clips[int(len(clips)/2)]['hue'] else: item = parts[4] parts = parts[5:] if parts and parts[0] in ("editor", "player"): parts = parts[1:] args = {} if parts and "?" in parts[-1]: part, arguments = parts[-1].split('?') parts[-1] = part args = dict( kv.split('=', 1) for kv in urllib.parse.unquote(arguments).split("&") ) if ',' in parts[0]: ts = [ox.parse_timecode(p) for p in parts[0].split(',')] if len(ts) >= 2: start, end = ts[-2:] else: logger.error("unable to decode in/out: %s", parts[0]) start = 0 end = 1000 request = { "action": "findClips", "data": { "query": { "conditions": [ {"key": "out", "value": start, "operator": ">"}, {"key": "in", "value": end, "operator": "<"}, ] }, "itemsQuery": { "conditions": [{"key": "id", "value": item, "operator": "=="}] }, "keys": ["id", "in", "out", "hue"], "range": [0, 5000], "sort": [ {"key": "in", "operator": "+"}, {"key": "out", "operator": "+"}, ] } } response = requests.post(url, json=request).json() items = response["data"]["items"] if items: hue = items[int(len(items)/2)]["hue"] else: annotation = '%s/%s' % (item, parts[0]) request = { "action": "getAnnotation", "data": { "id": annotation, "keys": ["hue"] } } response = requests.post(url, json=request).json() hue = response["data"]["hue"] if hue is not None: self.data["hue"] = hue return hue class Comment(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) item = models.ForeignKey(Item, related_name='comments', on_delete=models.CASCADE) text = models.TextField(default="") name = models.CharField(max_length=1024, blank=True) email = models.CharField(max_length=1024, blank=True) user = models.ForeignKey(User, null=True, related_name='comments', on_delete=models.CASCADE, blank=True) session_key = models.CharField(max_length=60, null=True, default=None, blank=True, editable=False) data = models.JSONField(default=dict, editable=False) published = models.DateTimeField(null=True, default=None, blank=True) class Meta: permissions = [ ("can_post_comment", "Can post comments without moderation") ] @property def is_published(self): return bool(self.published) def __str__(self): return '%s: %s' % (self.item, self.user) def save(self, *args, **kwargs): if self.user: self.name = self.user.username self.email = self.user.email super().save(*args, **kwargs) @property def date(self): now = timezone.now() difference = now - self.created if difference <= timedelta(minutes=1): return "just now" return '%(time)s ago' % {'time': timesince(self.created).split(', ')[0]} return self.created.strftime('%B %d, %Y at %H:%M') return self.created.strftime('%Y-%m-%d %H:%M') def json(self): data = {} if not self.user: data['name'] = '%s (guest)' % self.name else: data['name'] = self.name data['date'] = self.date data['text'] = self.text data['id'] = self.id data['published'] = self.is_published return data class Week(models.Model): created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) monday = models.DateField(unique=True) title = models.CharField(max_length=2048, blank=True, default="") byline = models.CharField(max_length=2048, blank=True, default="") published = models.DateTimeField(null=True, default=None, blank=True, editable=False) use_hue = models.BooleanField(default=False) is_break = models.BooleanField(default=False) break_notice = models.TextField(default="", blank=True) def __str__(self): return "%s (%s)" % (self.title, self.monday) def items(self): from datetime import date monday = timezone.make_aware(datetime.combine(self.monday, datetime.min.time()), timezone.get_default_timezone()) monday += timedelta(days=7) items, _ = Item.public(monday) return items.order_by('published') def background(self, now=None): if self.use_hue: colors = [] for item in self.items(): if now and item.published >= now: continue color = item.get_hue(update=True) if color: if not colors: colors.append(f'hsl({color}, 100%, 15%, 0.8)') else: colors.append(f'hsl({color}, 60%, 15%, 0.8)') return 'linear-gradient(to bottom, %s)' % ', '.join(colors) return ''