keywords

keywords.py

@@ -1,162 +1,300 @@
 from collections import defaultdict
 import re
 
-KEYWORDS = defaultdict(list)
-for line in '''
-1900-1947 (L)
-aerial shot (I, M, N)
-agriculture (B, I)
-air (C)
-air conditioner (H)
-anarchism (A)
-animal (Z)
-animism (A, S)
-aporia (A)
-archipelago (A, Z)
-ascending/descending (A)
-assimilation (A)
-authors/writing (G, I, L)
-balinese painting (P)
-bateson (P)
-barbarian (B)
-becoming-animal (X)
-body in water (L, W)
-bondage (B)
-boundary (B)
-buffalo (B, P)
-burial/grave (D)
-circle (C)
-colonialism (B, L, P, Q, T)
-construction (I)
-contagion (C, E)
-corruption (C)
-cosmology (C, M, S)
-cmp (G, L, T)
-mpc (G, L, T)
-mpaja (G, L, T)
-decay (D)
-deleuze/guattari (P)
-deleuzian diagram (P)
-diagram (A, H)
-dispersal/disaggregation (D)
-ecology (E)
-efficacy (E)
-emanation of power (S)
-espionage (G, L, T)
-evasion (E, Z)
-family (K)
-fanaticism (S)
-fiction (F)
-fire (V)
-first contact (X, Y)
-flight (F)
-flora/fauna (F)
-fluidity (F)
-foreigner (X, Y)
-forensics (D)
-forest (B, F, T, Z)
-friction (F)
-frontier (F)
-geography (G)
-ghost/spirit (G, S)
-gong (R)
-guerilla (G, L)
-heat (H)
-highland tribe (P)
-human remains (D)
-humidity (D, H)
-hydrology (H)
-identity (I)
-imitation (I)
-india/china (O)
-india bce (M)
-indian prisoners (T)
-inscription (I)
-intercourse (X)
-invulnerability (I)
-irrigation (H, I, P)
-japanese occupation (L, T)
-jellyfish (J, Z)
-kinship/kingship (K)
-landscape (H)
-legibility (L)
-magic/shamans (F, M, W, Y)
-malacca (U)
-mandala (M, S)
-manpower (M)
-map (M, S, T)
-metal (R)
-mobility (M)
-modern leader (C)
-modern nepotism (K, Z)
-money (C)
-mountain (A, Z)
-narcotics (N)
-narration (N)
-nationalism (N)
-network (M, S)
-obedience (S, Y)
-ocean (O, Q, U)
-opium (O)
-opium war (O)
-oppression (H, Z)
-orality (O)
-origin (O, U)
-outlaw (F)
-padi (P)
-peasants (B, P)
-periphery (P)
-piracy (A)
-plateau (P)
-politics (P)
-pontianak (V)
-pollution (J)
-queen (Q)
-question (Q)
-quotation (Q)
-rain (D, H)
-ritual (R, W)
-river (D, H, L, P)
-royalty (B, C, K, P, Q, S, U, Z)
-sea (S)
-shadow puppet (B)
-shape-shifter (J, L, Z)
-singapore (U)
-singapore before 1900 (T)
-slavery (S, T)
-society/state (S)
-soil (D)
-soul (S)
-sumatra (U)
-survey (M, N, T)
-sweat/labor (H)
-swidden agriculture (P)
-tamil tigers (W)
-tattoo (I)
-tiger (B, P, T, W)
-trade (H)
-transmission (C, G)
-transmission of ideology (E)
-underwater (Q)
-unreadable sign (I, L)
-utama (U)
-vagina (V)
-vampire (V)
-voice (V)
-water (A, H, T, W, Z)
-water kingdom (Q, U)
-weretiger (W, Z)
-women (Q)
-women with white males (Q)
-xenophilia/xenophopia (X)
-yielding (Y)
-zomia (A, J, Z)
-zoophilia (Z)
-zombie (Z)
-
-'''.strip().split('\n'):
-    m = re.compile('(.*?) \((.*?)\)').findall(line)
-    term = m[0][0]
-    for id in m[0][1].split(','):
-        id = id.strip()
-        KEYWORDS[id].append(term)
+KEYWORDS = {
+  "A": [
+    "anarchism",
+    "animism",
+    "aporia",
+    "archipelago",
+    "ascending/descending",
+    "assimilation",
+    "diagram",
+    "mountain",
+    "piracy",
+    "water",
+    "zomia"
+  ],
+  "B": [
+    "agriculture",
+    "barbarian",
+    "bondage",
+    "boundary",
+    "buffalo",
+    "colonialism",
+    "forest",
+    "peasants",
+    "royalty",
+    "shadow puppet",
+    "tiger"
+  ],
+  "C": [
+    "air",
+    "circle",
+    "contagion",
+    "corruption",
+    "cosmology",
+    "modern leader",
+    "money",
+    "royalty",
+    "transmission"
+  ],
+  "D": [
+    "burial/grave",
+    "decay",
+    "dispersal", "disaggregation",
+    "forensics",
+    "human remains",
+    "humidity",
+    "rain",
+    "river",
+    "soil"
+  ],
+  "E": [
+    "evasion",
+    "epidemic",
+    "contagion",
+    "ecology",
+    "efficacy",
+    "transmission of ideology"
+  ],
+  "F": [
+    "fiction",
+    "flight",
+    "flora", "fauna",
+    "fluidity",
+    "forest",
+    "friction",
+    "frontier",
+    "magic/shamans",
+    "outlaw"
+  ],
+  "G": [
+    "authors/writing",
+    "cmp",
+    "mpc",
+    "mpaja",
+    "espionage",
+    "geography",
+    "ghost", "spirit",
+    "guerilla",
+    "transmission"
+  ],
+  "H": [
+    "air conditioner",
+    "diagram",
+    "heat",
+    "humidity",
+    "hydrology",
+    "irrigation",
+    "landscape",
+    "oppression",
+    "rain",
+    "river",
+    "sweat", "labor",
+    "trade",
+    "water"
+  ],
+  "I": [
+    "aerial shot",
+    "agriculture",
+    "authors/writing",
+    "construction",
+    "identity",
+    "imitation",
+    "inscription",
+    "invulnerability",
+    "irrigation",
+    "tattoo",
+    "unreadable sign"
+  ],
+  "J": [
+    "jellyfish",
+    "pollution",
+    "shape-shifter",
+    "zomia"
+  ],
+  "K": [
+    "family",
+    "kinship/kingship",
+    "modern nepotism",
+    "royalty"
+  ],
+  "L": [
+    "1900-1947",
+    "authors/writing",
+    "body in water",
+    "colonialism",
+    "cmp",
+    "mpc",
+    "mpaja",
+    "espionage",
+    "guerilla",
+    "japanese occupation",
+    "legibility",
+    "river",
+    "shape-shifter",
+    "unreadable sign"
+  ],
+  "M": [
+    "aerial shot",
+    "cosmology",
+    "india bce",
+    "magic/shamans",
+    "mandala",
+    "manpower",
+    "map",
+    "mobility",
+    "network",
+    "survey"
+  ],
+  "N": [
+    "aerial shot",
+    "narcotics",
+    "narration",
+    "nationalism",
+    "survey"
+  ],
+  "O": [
+    "india", "china",
+    "ocean",
+    "opium",
+    "opium war",
+    "orality",
+    "origin"
+  ],
+  "P": [
+    "balinese painting",
+    "bateson",
+    "buffalo",
+    "colonialism",
+    "deleuze/guattari",
+    "deleuzian diagram",
+    "highland tribe",
+    "irrigation",
+    "padi",
+    "peasants",
+    "periphery",
+    "plateau",
+    "politics",
+    "river",
+    "royalty",
+    "swidden agriculture",
+    "tiger"
+  ],
+  "Q": [
+    "colonialism",
+    "ocean",
+    "queen",
+    "question",
+    "quotation",
+    "royalty",
+    "underwater",
+    "water kingdom",
+    "women",
+    "women with white males"
+  ],
+  "R": [
+    "gong",
+    "metal",
+    "ritual"
+  ],
+  "S": [
+    "animism",
+    "cosmology",
+    "emanation of power",
+    "fanaticism",
+    "ghost", "spirit",
+    "mandala",
+    "map",
+    "network",
+    "obedience",
+    "royalty",
+    "sea",
+    "slavery",
+    "society", "state",
+    "soul"
+  ],
+  "T": [
+    "tiger",
+    "tiger",
+    "tiger",
+    "tiger",
+    "tiger",
+    "tiger",
+    "tiger",
+    "colonialism",
+    "cpm/mcp/mpaja",
+    "espionage",
+    "forest",
+    "indian prisoners",
+    "japanese occupation",
+    "map",
+    "singapore before 1900",
+    "slavery",
+    "survey",
+    "theodolite",
+    "water"
+  ],
+  "U": [
+    "malacca",
+    "ocean",
+    "origin",
+    "royalty",
+    "singapore",
+    "sumatra",
+    "utama",
+    "water kingdom"
+  ],
+  "V": [
+    "fire",
+    "pontianak",
+    "vagina",
+    "vampire",
+    "voice"
+  ],
+  "W": [
+    "weretiger",
+    "weretiger",
+    "weretiger",
+    "water",
+    "water",
+    "body in water",
+    "magic/shamans",
+    "ritual",
+    "tamil tigers",
+    "tiger",
+  ],
+  "X": [
+    "becoming-animal",
+    "first contact",
+    "foreigner",
+    "intercourse",
+    "xenophilia/xenophopia",
+  ],
+  "Y": [
+    "first contact",
+    "foreigner",
+    "magic/shamans",
+    "obedience",
+    "youth",
+    "yielding"
+  ],
+  "Z": [
+    "animal",
+    "archipelago",
+    "evasion",
+    "forest",
+    "jellyfish",
+    "modern nepotism",
+    "mountain",
+    "oppression",
+    "royalty",
+    "shape-shifter",
+    "water",
+    "weretiger",
+    "zomia",
+    "zoophilia",
+    "zombie"
+  ]
+}

render.py

@@ -1,8 +1,10 @@
 #!/usr/bin/python3
 import os
+import sys
 import json
 import subprocess
 import string
+from glob import glob
 
 from pi import random
 from keywords import KEYWORDS
@@ -44,9 +46,20 @@ def get_path(id):
             'range': [0, 1]
         })['data']['items'][0]
         path = os.path.join('cache', '%s.%s' % (info['id'], info['extension']))
+        h = info['id']
+        source = '/srv/pandora/data/media/%s/%s/%s/%s/data.*' % (
+            h[:2], h[2:4], h[4:6], h[6:]
+        )
+        source = glob(source)[0]
         if not os.path.exists(path):
+            if not os.path.exists(source):
+                print('WTF', source)
+                sys.exit(1)
+            os.symlink(source, path)
+            '''
             url = '%s/%s/download/source/' % (base_url, id)
             print('get video', url)
+            '''
         PATHS[id] = path
         with open('PATHS.json', 'w') as fd:
             json.dump(PATHS, fd, indent=4, sort_keys=True)
@@ -77,18 +90,18 @@ def get_clips(tag):
 
 def random_choice(seq, items):
     n = n_ = len(items) - 1
-    print('len', n)
+    #print('len', n)
     if n == 0:
         return items[0]
     r = seq()
     base = 10
     while n > 10:
         n /= 10
-        print(r)
+        #print(r)
         r += seq()
         base += 10
     r = int(n_ * r / base)
-    print('result', r, items)
+    #print('result', r, items)
     return items[r]
 
 def splitint(number, by):
@@ -114,14 +127,14 @@ def filter_clips(clips, duration, max_duration=0):
             clip['tag'] = tag
             clips_.append(clip)
     clips_.sort(key=lambda c: c['duration'])
-    print(clips_)
+    #print(clips_)
     size = splitint(len(clips_), 10)
     p = 0
     for i in range(10):
         buckets[i+1] = clips_[p:+p+size[i]]
         p += size[i]
     clips_ = {}
-    print(buckets[duration])
+    #print(buckets[duration])
     for clip in buckets[duration]:
         if clip['tag'] not in clips_:
             clips_[clip['tag']] = []
@@ -146,9 +159,13 @@ def sequence(seq, letter):
 
         max_duration = MAX_DURATION - duration
         clips_n = filter_clips(clips, n, max_duration)
-        tags_n = list(sorted(clips_n.keys()))
+        tags_n = []
+        for tag in tags:
+            if tag in clips_n:
+                tags_n.append(tag)
         if not tags_n:
             print('NO tags for', letter, n)
+            sys.exit(1)
         tag = random_choice(seq, tags_n)
         #if 'tiger' in tags_n:
         #    tag = 'tiger'
@@ -159,6 +176,7 @@ def sequence(seq, letter):
         if seq() == 0:
             clip['black'] = True
     position = last_text = 0
+    tags_text = []
     while position < duration:
         n = seq()
         if n == 0:
@@ -168,8 +186,12 @@ def sequence(seq, letter):
             if n == 0:
                 n = 10
             n = min(n, duration-position)
+            if not tags_text:
+                tags_text = list(sorted(set(tags)))
+            ttag = random_choice(seq, tags_text)
+            tags_text.remove(ttag)
             text = {
-                'text': random_choice(seq, tags_n),
+                'text': ttag,
                 'duration': n
             }
             result['text'].append(text)
@@ -185,13 +207,23 @@ def sequence(seq, letter):
 
 if __name__ == '__main__':
     for n in range(10):
-        seq = random(n)
-        #for letter in string.ascii_uppercase:
-        for letter in ('T', 'W'):
+        seq = random(n * 1000)
+        #for letter in ('T', 'W'):
+        for letter in string.ascii_uppercase:
             r = sequence(seq, letter)
-            tjson = 'output/%s%d.json' % (letter, n)
-            with open(tjson, 'w') as fd:
-                json.dump(r, fd, indent=4, sort_keys=True)
-            print(json.dumps(r, indent=4, sort_keys=True))
+            tjson = 'output/%02d/%s.json' % (n, letter)
+            folder = os.path.dirname(tjson)
+            if not os.path.exists(folder):
+                ox.makedirs(folder)
+            if os.path.exists(tjson):
+                with open(tjson, 'r') as fd:
+                    old = fd.read()
+            else:
+                old = None
+            current = json.dumps(r, indent=4, sort_keys=True)
+            #print(current)
             #print(sum([c['duration'] for c in r['clips']]))
+            if current != old:
+                with open(tjson, 'w') as fd:
+                    fd.write(current)
                 subprocess.call(['./render_mlt.py', tjson])

render_mlt.py

@@ -67,7 +67,7 @@ for clip in data['clips']:
         frames = int(clip['duration'] * fps)
         add_color(video, 'black', frames)
     else:
-        print(clip['duration'], clip['path'])
+        #print(clip['duration'], clip['path'])
         if not os.path.exists(clip['path']):
             print(clip['path'], 'is missing')
             sys.exit(1)
@@ -75,6 +75,7 @@ for clip in data['clips']:
         in_ = int(clip['in'] * fps)
         frames = int(clip['duration'] * fps)
         add_clip(video, clip['path'], in_, frames)
+
 for clip in data['text']:
     if clip.get('blank'):
         frames = int(clip['duration'] * fps)
@@ -86,7 +87,7 @@ for clip in data['text']:
 multitrack.connect(video, 0)
 multitrack.connect(overlay, 1)
 composite = mlt.Transition(profile, "composite")
-#composite.set('fill', 0)
+#composite.set('fill', 1)
 
 tractor.plant_transition(composite)