diff --git a/API.md b/API.md new file mode 100644 index 0000000..0cbaaaa --- /dev/null +++ b/API.md @@ -0,0 +1,155 @@ +## API +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +### Documentation +Each instance of Pan.do/ra comes with its build in API documentation. You find it by navigation to /api/ with a browser. +I.e. current API can be found at + +### Using the API +To use the API you can use Ox.App if you use Ox.js, or look at [pandora_client](https://wiki.0x2620.org/browser/pandora_client/pandora_client/*init*.py#L331) for how this can be used from python. Eventually we hope to describe this section in more detail. + + + +### Examples + + +#### PHP +``` +url = $url; + } + public function _request($action, $data) { + $content = array( + 'action' => $action, + 'data' => json_encode($data) + ); + $options = array( + 'http' => array( + 'header' => "Content-type: application/x-www-form-urlencoded\r\n", + 'method' => 'POST', + 'content' => http_build_query($content), + ), + ); + $context = stream_context_create($options); + $result = json_decode(file_get_contents($this->url, false, $context)); + return $result; + } +} +l +$api = new API('http://archive.arabdigitalexpression.org/api/'); + +$result = $api->_request('find', array( + "query"=>array( + "conditions"=>array(array("key"=>"*", "value"=>"paris", "operator"=>'=')) + ), + "keys"=>array("title", 'id'), + "range"=> array(0, 10), + "sort"=>array(array("key"=>"title", "operator"=>"+")) +)); + +var_dump($result); + +``` + +#### JavaScript +``` + + + + + + + + +``` + +Or using OxJS +``` + + + + + + + + + +``` + +#### Python + +Using python-ox (install with **easy_install ox**) + +``` +import ox + +api = ox.API('http://archive.arabdigitalexpression.org/api/') + +item = api.find({'query':{conditions:[{'key': '*', 'value':'test', 'operator':'='}], 'operator':'&'}, 'keys': ['location']}) + +``` + +Add all items in list 'j:Example' to groups a, b and c +``` +#!/usr/bin/python +import ox + +username=ADD_USER +password=ADD_PASSWORD +groups = ['a', 'b', 'c'] +list_id = 'j:Example' + +api = ox.API('http://kjc-sv030.kjc.uni-heidelberg.de/api/') +api.signin(username=username, password=password) +for i in api.find({'query': { + 'conditions': [{'key': 'list', 'value': list_id}]}, + 'keys': ['id', 'groups'], + 'range': [0, 10000] +})['data']['items']: + if i['groups'] != groups: + print id + api.edit({'id': i['id'], 'groups': groups}) +``` \ No newline at end of file diff --git a/DatabaseUpdate.md b/DatabaseUpdate.md new file mode 100644 index 0000000..2b3dd16 --- /dev/null +++ b/DatabaseUpdate.md @@ -0,0 +1,43 @@ +## Upgrading Database to South +first time you update from a version without South you have to manually sync your db and after that initialize your database and fake all previous south transactions: + + 1. if your repositroy revision is below 2327, you want to update to 2327, migrate to south and update to latest version after that +``` +bzr pull -r2327 +cd src/python-ox +bzr pull +``` + 2. manualy sync database to current layout(see below) + 3. make sure south is installed +``` +cd /srv/pandora +./bin/pip install South +``` + + 4. sync south db +``` +cd /srv/pandora/pandora +./manage.py syncdb +``` + 5. populate south history +``` +./manage.py migrate --all --fake +``` + +## Manually Syncing Database +First check for changes in the database: +``` +cd /srv/pandora/pandora +./manage.py sqldiff -a +``` +If you have any you can pipe the diff into dbshell to apply them. make sure nothing gets dropped that contains data. +``` +cd /srv/pandora/pandora +./manage.py sqldiff -a | ./manage.py dbshell +``` +Sometimes it is required to set defaults for rows. In that case you have to open dbshell and set them manually: +``` +./manage.py dbshell +UPDATE archive_file SET info = '{}' WHERE info IS NULL; +UPDATE archive_file SET path_info = '{}' WHERE path_info IS NULL; +``` \ No newline at end of file diff --git a/IMDb.md b/IMDb.md new file mode 100644 index 0000000..ba568fc --- /dev/null +++ b/IMDb.md @@ -0,0 +1,126 @@ +## IMDb Parser +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +Open Issues: + + * some feelds need html stripped, others not + * tv shows have " around title + * lists of type, i.e. release_date can be a list of dates with current parser + +this is now implemented in python-ox +[see here](http://code.0x2620.org/python-ox/annotate/head%3A/ox/web/imdb.py) + +``` +{ + 'cast': { + 'page': 'combined', + 're': '.*?>(.*?).*?(.*?)', + 'type': 'list' + }, + 'cinematographers': { + 'page': 'combined', + 're': [ + 'Cinematography by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'countries': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'directors': { + 'page': 'combined', + 're': [ + 'Directed by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'editors': { + 'page': 'combined', + 're': [ + 'Film Editing by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'filming_locations': { + 'page': 'locations', + 're': '(.*?)', + 'type': 'list' + }, + 'genres': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'keywords': { + 'page': 'keywords', + 're': '(.*?)', + 'type': 'list' + }, + 'languages': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'poster_id': { + 'page': 'combined', + 're': '/primary-photo/media/rm(.*?)/tt', + 'type': 'list' + }, + 'poster_ids': { + 'page': 'posters', + 're': '/unknown-thumbnail/media/rm(.*?)/tt', + 'type': 'list' + }, + 'producers': { + 'page': 'combined', + 're': [ + 'Produced by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'rating': { + 'page': 'combined', + 're': '
.*?(.*?)/10', + 'type': 'float' + }, + 'release_date': { + 'page': 'releaseinfo', + 're': '.*? ', + 'type': 'date' + }, + 'title': { + 'page': 'combined', + 're': '

(.*?) ', + 'type': 'list' + }, + 'trivia': { + 'page': 'trivia', + 're': '
(.*?)
', + 'type': 'list', + }, + 'votes': { + 'page': 'combined', + 're': '
(.*?) votes', + 'type': 'int' + }, + 'writers': { + 'page': 'combined', + 're': [ + 'Writing credits(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'year': { + 'page': 'combined', + 're': '', + 'type': 'int' + } +} +``` \ No newline at end of file diff --git a/MultilingualAnnotations.md b/MultilingualAnnotations.md new file mode 100644 index 0000000..aa0c44f --- /dev/null +++ b/MultilingualAnnotations.md @@ -0,0 +1,4 @@ +## Multilingual Annotations + +To mark out languages in annotations, you can use the html lang attribute while writing an annotation or select a language during annotation import. +The attribute can be added to any html element. If no element is used, wrap the annotation inside a span tag: `...` \ No newline at end of file diff --git a/MultipleAudioTracks.md b/MultipleAudioTracks.md new file mode 100644 index 0000000..0668263 --- /dev/null +++ b/MultipleAudioTracks.md @@ -0,0 +1,6 @@ +## Multiple Audio Tracks + +pan.do/ra supports items with multiple audio tracks. You can upload videos with multiple audio tracks, i.e. in mkv format. Another option is to upload each language as a new video and instead of parts, set the language attribute to each video (in the item media view). + +Instead of iso 2 letter language codes you can also use an alternative track name. I.e. Director's Commentary. + diff --git a/Places.md b/Places.md new file mode 100644 index 0000000..2a1010f --- /dev/null +++ b/Places.md @@ -0,0 +1,10 @@ + * name: 'Camp Rooftop' + * aliases: ['Pirate Cinema Bombay'] + * geoname: 'Camp Rooftop, Bandra, Bombay, Maharashtra, India' + * geoname_reverse: 'India, Maharashtra, Bombay, Bandra, Camp Rooftop' + * center_lat + * center_lng + * sw_lat + * sw_lng + * ne_lat + * ne_lng diff --git a/QuerySyntax.md b/QuerySyntax.md new file mode 100644 index 0000000..03d8666 --- /dev/null +++ b/QuerySyntax.md @@ -0,0 +1,129 @@ +## Query Syntax +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +Simple query +``` +query: { + conditions: [ + { + key: "year", + value: [1970, 1980], + operator: "!-" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" +} +``` + +Complex query +``` +query: { + conditions: [ + { + conditions: [ + { + key: "year", + value: 1960, + operator: "<" + }, + { + key: "year", + value: 1960, + operator: ">" + }, + { + key: "title", + value: "sex", + operator: "" + }, + ], + operator: "|" + }, + { + key: "director", + value: "jean-luc godard", + operator: "=" + } + ], + operator: "&" +} +``` + +Objects +``` +query: {conditions, operator} +conditions: [query|condition] +condition: {key, value, operator} +``` + +Operators +``` +string +x: contains +!x: does not contain +^x: starts with +!^: does not start with +x$: ends with +!$: does not end with +^x$: is +!^x$: is not + +number +x: is +!x: is not +x: is greater than or equal to +x-x: is between +!x-x: not between + +conditions +a,b: a and b +a|b: a or b +``` + +Examples +``` +foo +keyword:foo +year:1980,hello,country:usa +year:1980,hello,country:!usa +title:!^the +title:^the$ +title:^100%24$ +year:>1960,year:<1970 +year:<1960|year:>1970|title:sex +profit:-1000000-1000000 +profit:-1000000--500000 +releasedate:1970-10-1970-12 +releasedate:1970-1972-12-13 +[year:<1960|year:>1970|title:sex],director:godard // group godard will be selected +[year:<1960|year:>1970|title:sex],[director:godard|director:scorsese] // groups godard and scorsese will be selected +director:godard,country:france,year:1967 // groups godard, france and 1967 will be selected +list:mylist,director:hitchcock +list:otheruser.publiclist,director:hitchcock +list:0xdb.timelines,director:hitchcock +``` + +Recursion +``` +list:you.yourlist,list:!mylist (both dynamic) +... save as wishlist +... edit mylist to be wishlist +... now wishlist is: you.yourlist,!wishlist -> recursion +so if you change a dynamic list to contain a reference to itself, +you will get a recursion error. +which means that we'll need to add to the API a request like +function:norecursion,data:mydynamiclist +that returns a list of recursion-safe public dynamic lists +``` + +URLs +``` +0xdb.org/#find=title:foo,director:!bar&sort=country,-year&view=icon,poster +0xdb.org/?find=title:foo,director:!bar&sort=country,-year&view=icon,poster // do we need this for search engines? +``` \ No newline at end of file diff --git a/RenamingFiles.md b/RenamingFiles.md new file mode 100644 index 0000000..1dd5046 --- /dev/null +++ b/RenamingFiles.md @@ -0,0 +1,45 @@ +## Semantic Meaning of Files + +Each movie has a number of meta files, each of which has a number of file instances (paths in archives). + +A meta file can have the following meaning: + +- Movie File (avi/mkv/...) + +- Movie File (avi/mkv/...), Part X + +- Subtitle File (srt/sub/idx/rar) + +- Subtitle File (srt/sub/idx/rar), Part X + +- Other File (mp3/txt/...) + +- Other File (mp3/txt/...), Part X + +## Potential Errors and Warnings + +Before files in an archive can be automatically renamed, we have to check for the following conditions: + +Errors: + +- "Concurrent Files" (multiple files with the same meaning and extension) + +Warnings: + +- "Multiple Files" (movie files with same meaning, but different extension) + +- "Incorrect Subtitles" (there are subtitles, but not every movie file has subtitles, or not every subtitle file has a movie file) + +- "Incomplete Subtitles" (idx without sub, or sub without idx) + +- "Multiple Versions" (both non-part and part files) + +- "Multiple Names" (different strings in Title.Version Name.Part X.ext) + +- "Incorrect Parts" (parts are not 1, 2, 3 ... n) + +## Renaming Files + +Once the errors have been resolved, files can be renamed to: + +L/Lastname, Firstname/The Title (Year)/The Title[.Version Name][.Part X].ext \ No newline at end of file diff --git a/UseCases.md b/UseCases.md new file mode 100644 index 0000000..2278d4a --- /dev/null +++ b/UseCases.md @@ -0,0 +1,60 @@ +## Use Cases +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +### Visitor + * search for a specific video + * browse videos + * embed clip on own page + * download full video + * download selected clip from video + +### registered User + * add annotation + * manage videos in lists + * use browser extension to manage local videos + +### Contributor(is this same as user?) + + * create annotations before upload + * upload video and annotations + * add new annotations + * add geo data to location names + * manage videos in lists + * add keywords and categories intelligently (?) + * get some data about downloads, views of their video. (?) + + +### VIP + + * see more info about videos + +### Admin + + * change data for users, videos, layers + * statistics + * remove video + * manipulate other information in terminal + * run scripts to update information on all videos + * mange featured lists + * manage subdomains + * change ownership for all layers of one video + * change ownership for all layers of one video belonging to user a to user b + +### API/Bots/Embedding + * embed clips from videos on other pages + * embed maps with clips on other pages + * use api to build custom interfaces + * make sure POST requests are Cross-site scripting (XSS) save + * implement something like JSONP so that Api works when user can only use JS. (?) + * Allow user to select in-point, out-point and tracks to display, and generate an or