From 537d4ab5b98d03e2570465dfca90f73cdb7f6b19 Mon Sep 17 00:00:00 2001 From: j Date: Sun, 2 Jul 2023 12:57:13 +0530 Subject: [PATCH] migrate wiki --- API.md | 155 +++++++++++++++++++ DatabaseUpdate.md | 43 ++++++ IMDb.md | 126 ++++++++++++++++ MultilingualAnnotations.md | 4 + MultipleAudioTracks.md | 6 + Places.md | 10 ++ QuerySyntax.md | 129 ++++++++++++++++ RenamingFiles.md | 45 ++++++ UseCases.md | 60 ++++++++ config.md | 20 +++ configuration.md | 67 +++++++++ embed.md | 16 ++ filenames.md | 73 +++++++++ groups.md | 14 ++ ids.md | 39 +++++ install.md | 8 + ldap.md | 46 ++++++ media.md | 26 ++++ metadata.md | 270 +++++++++++++++++++++++++++++++++ objects.md | 75 ++++++++++ objects/Item.md | 14 ++ objects/Place.md | 21 +++ permissions.md | 21 +++ posterservice.md | 11 ++ services.md | 34 +++++ shell.md | 37 +++++ ssl.md | 64 ++++++++ text.md | 157 ++++++++++++++++++++ todo.md | 74 ++++++++++ urls.md | 296 +++++++++++++++++++++++++++++++++++++ video.md | 25 ++++ volume.md | 11 ++ 32 files changed, 1997 insertions(+) create mode 100644 API.md create mode 100644 DatabaseUpdate.md create mode 100644 IMDb.md create mode 100644 MultilingualAnnotations.md create mode 100644 MultipleAudioTracks.md create mode 100644 Places.md create mode 100644 QuerySyntax.md create mode 100644 RenamingFiles.md create mode 100644 UseCases.md create mode 100644 config.md create mode 100644 configuration.md create mode 100644 embed.md create mode 100644 filenames.md create mode 100644 groups.md create mode 100644 ids.md create mode 100644 install.md create mode 100644 ldap.md create mode 100644 media.md create mode 100644 metadata.md create mode 100644 objects.md create mode 100644 objects/Item.md create mode 100644 objects/Place.md create mode 100644 permissions.md create mode 100644 posterservice.md create mode 100644 services.md create mode 100644 shell.md create mode 100644 ssl.md create mode 100644 text.md create mode 100644 todo.md create mode 100644 urls.md create mode 100644 video.md create mode 100644 volume.md diff --git a/API.md b/API.md new file mode 100644 index 0000000..0cbaaaa --- /dev/null +++ b/API.md @@ -0,0 +1,155 @@ +## API +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +### Documentation +Each instance of Pan.do/ra comes with its build in API documentation. You find it by navigation to /api/ with a browser. +I.e. current API can be found at + +### Using the API +To use the API you can use Ox.App if you use Ox.js, or look at [pandora_client](https://wiki.0x2620.org/browser/pandora_client/pandora_client/*init*.py#L331) for how this can be used from python. Eventually we hope to describe this section in more detail. + + + +### Examples + + +#### PHP +``` +url = $url; + } + public function _request($action, $data) { + $content = array( + 'action' => $action, + 'data' => json_encode($data) + ); + $options = array( + 'http' => array( + 'header' => "Content-type: application/x-www-form-urlencoded\r\n", + 'method' => 'POST', + 'content' => http_build_query($content), + ), + ); + $context = stream_context_create($options); + $result = json_decode(file_get_contents($this->url, false, $context)); + return $result; + } +} +l +$api = new API('http://archive.arabdigitalexpression.org/api/'); + +$result = $api->_request('find', array( + "query"=>array( + "conditions"=>array(array("key"=>"*", "value"=>"paris", "operator"=>'=')) + ), + "keys"=>array("title", 'id'), + "range"=> array(0, 10), + "sort"=>array(array("key"=>"title", "operator"=>"+")) +)); + +var_dump($result); + +``` + +#### JavaScript +``` + + + + + + + + +``` + +Or using OxJS +``` + + + + + + + + + +``` + +#### Python + +Using python-ox (install with **easy_install ox**) + +``` +import ox + +api = ox.API('http://archive.arabdigitalexpression.org/api/') + +item = api.find({'query':{conditions:[{'key': '*', 'value':'test', 'operator':'='}], 'operator':'&'}, 'keys': ['location']}) + +``` + +Add all items in list 'j:Example' to groups a, b and c +``` +#!/usr/bin/python +import ox + +username=ADD_USER +password=ADD_PASSWORD +groups = ['a', 'b', 'c'] +list_id = 'j:Example' + +api = ox.API('http://kjc-sv030.kjc.uni-heidelberg.de/api/') +api.signin(username=username, password=password) +for i in api.find({'query': { + 'conditions': [{'key': 'list', 'value': list_id}]}, + 'keys': ['id', 'groups'], + 'range': [0, 10000] +})['data']['items']: + if i['groups'] != groups: + print id + api.edit({'id': i['id'], 'groups': groups}) +``` \ No newline at end of file diff --git a/DatabaseUpdate.md b/DatabaseUpdate.md new file mode 100644 index 0000000..2b3dd16 --- /dev/null +++ b/DatabaseUpdate.md @@ -0,0 +1,43 @@ +## Upgrading Database to South +first time you update from a version without South you have to manually sync your db and after that initialize your database and fake all previous south transactions: + + 1. if your repositroy revision is below 2327, you want to update to 2327, migrate to south and update to latest version after that +``` +bzr pull -r2327 +cd src/python-ox +bzr pull +``` + 2. manualy sync database to current layout(see below) + 3. make sure south is installed +``` +cd /srv/pandora +./bin/pip install South +``` + + 4. sync south db +``` +cd /srv/pandora/pandora +./manage.py syncdb +``` + 5. populate south history +``` +./manage.py migrate --all --fake +``` + +## Manually Syncing Database +First check for changes in the database: +``` +cd /srv/pandora/pandora +./manage.py sqldiff -a +``` +If you have any you can pipe the diff into dbshell to apply them. make sure nothing gets dropped that contains data. +``` +cd /srv/pandora/pandora +./manage.py sqldiff -a | ./manage.py dbshell +``` +Sometimes it is required to set defaults for rows. In that case you have to open dbshell and set them manually: +``` +./manage.py dbshell +UPDATE archive_file SET info = '{}' WHERE info IS NULL; +UPDATE archive_file SET path_info = '{}' WHERE path_info IS NULL; +``` \ No newline at end of file diff --git a/IMDb.md b/IMDb.md new file mode 100644 index 0000000..ba568fc --- /dev/null +++ b/IMDb.md @@ -0,0 +1,126 @@ +## IMDb Parser +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +Open Issues: + + * some feelds need html stripped, others not + * tv shows have " around title + * lists of type, i.e. release_date can be a list of dates with current parser + +this is now implemented in python-ox +[see here](http://code.0x2620.org/python-ox/annotate/head%3A/ox/web/imdb.py) + +``` +{ + 'cast': { + 'page': 'combined', + 're': '.*?>(.*?).*?(.*?)', + 'type': 'list' + }, + 'cinematographers': { + 'page': 'combined', + 're': [ + 'Cinematography by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'countries': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'directors': { + 'page': 'combined', + 're': [ + 'Directed by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'editors': { + 'page': 'combined', + 're': [ + 'Film Editing by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'filming_locations': { + 'page': 'locations', + 're': '(.*?)', + 'type': 'list' + }, + 'genres': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'keywords': { + 'page': 'keywords', + 're': '(.*?)', + 'type': 'list' + }, + 'languages': { + 'page': 'combined', + 're': '(.*?)', + 'type': 'list' + }, + 'poster_id': { + 'page': 'combined', + 're': '/primary-photo/media/rm(.*?)/tt', + 'type': 'list' + }, + 'poster_ids': { + 'page': 'posters', + 're': '/unknown-thumbnail/media/rm(.*?)/tt', + 'type': 'list' + }, + 'producers': { + 'page': 'combined', + 're': [ + 'Produced by(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'rating': { + 'page': 'combined', + 're': '
.*?(.*?)/10', + 'type': 'float' + }, + 'release_date': { + 'page': 'releaseinfo', + 're': '.*? ', + 'type': 'date' + }, + 'title': { + 'page': 'combined', + 're': '

(.*?) ', + 'type': 'list' + }, + 'trivia': { + 'page': 'trivia', + 're': '
(.*?)
', + 'type': 'list', + }, + 'votes': { + 'page': 'combined', + 're': '(.*?) votes', + 'type': 'int' + }, + 'writers': { + 'page': 'combined', + 're': [ + 'Writing credits(.*?)', + '(.*?)' + ], + 'type': 'list' + }, + 'year': { + 'page': 'combined', + 're': '', + 'type': 'int' + } +} +``` \ No newline at end of file diff --git a/MultilingualAnnotations.md b/MultilingualAnnotations.md new file mode 100644 index 0000000..aa0c44f --- /dev/null +++ b/MultilingualAnnotations.md @@ -0,0 +1,4 @@ +## Multilingual Annotations + +To mark out languages in annotations, you can use the html lang attribute while writing an annotation or select a language during annotation import. +The attribute can be added to any html element. If no element is used, wrap the annotation inside a span tag: `...` \ No newline at end of file diff --git a/MultipleAudioTracks.md b/MultipleAudioTracks.md new file mode 100644 index 0000000..0668263 --- /dev/null +++ b/MultipleAudioTracks.md @@ -0,0 +1,6 @@ +## Multiple Audio Tracks + +pan.do/ra supports items with multiple audio tracks. You can upload videos with multiple audio tracks, i.e. in mkv format. Another option is to upload each language as a new video and instead of parts, set the language attribute to each video (in the item media view). + +Instead of iso 2 letter language codes you can also use an alternative track name. I.e. Director's Commentary. + diff --git a/Places.md b/Places.md new file mode 100644 index 0000000..2a1010f --- /dev/null +++ b/Places.md @@ -0,0 +1,10 @@ + * name: 'Camp Rooftop' + * aliases: ['Pirate Cinema Bombay'] + * geoname: 'Camp Rooftop, Bandra, Bombay, Maharashtra, India' + * geoname_reverse: 'India, Maharashtra, Bombay, Bandra, Camp Rooftop' + * center_lat + * center_lng + * sw_lat + * sw_lng + * ne_lat + * ne_lng diff --git a/QuerySyntax.md b/QuerySyntax.md new file mode 100644 index 0000000..03d8666 --- /dev/null +++ b/QuerySyntax.md @@ -0,0 +1,129 @@ +## Query Syntax +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +Simple query +``` +query: { + conditions: [ + { + key: "year", + value: [1970, 1980], + operator: "!-" + }, + { + key: "country", + value: "f", + operator: "^" + } + ], + operator: "&" +} +``` + +Complex query +``` +query: { + conditions: [ + { + conditions: [ + { + key: "year", + value: 1960, + operator: "<" + }, + { + key: "year", + value: 1960, + operator: ">" + }, + { + key: "title", + value: "sex", + operator: "" + }, + ], + operator: "|" + }, + { + key: "director", + value: "jean-luc godard", + operator: "=" + } + ], + operator: "&" +} +``` + +Objects +``` +query: {conditions, operator} +conditions: [query|condition] +condition: {key, value, operator} +``` + +Operators +``` +string +x: contains +!x: does not contain +^x: starts with +!^: does not start with +x$: ends with +!$: does not end with +^x$: is +!^x$: is not + +number +x: is +!x: is not +x: is greater than or equal to +x-x: is between +!x-x: not between + +conditions +a,b: a and b +a|b: a or b +``` + +Examples +``` +foo +keyword:foo +year:1980,hello,country:usa +year:1980,hello,country:!usa +title:!^the +title:^the$ +title:^100%24$ +year:>1960,year:<1970 +year:<1960|year:>1970|title:sex +profit:-1000000-1000000 +profit:-1000000--500000 +releasedate:1970-10-1970-12 +releasedate:1970-1972-12-13 +[year:<1960|year:>1970|title:sex],director:godard // group godard will be selected +[year:<1960|year:>1970|title:sex],[director:godard|director:scorsese] // groups godard and scorsese will be selected +director:godard,country:france,year:1967 // groups godard, france and 1967 will be selected +list:mylist,director:hitchcock +list:otheruser.publiclist,director:hitchcock +list:0xdb.timelines,director:hitchcock +``` + +Recursion +``` +list:you.yourlist,list:!mylist (both dynamic) +... save as wishlist +... edit mylist to be wishlist +... now wishlist is: you.yourlist,!wishlist -> recursion +so if you change a dynamic list to contain a reference to itself, +you will get a recursion error. +which means that we'll need to add to the API a request like +function:norecursion,data:mydynamiclist +that returns a list of recursion-safe public dynamic lists +``` + +URLs +``` +0xdb.org/#find=title:foo,director:!bar&sort=country,-year&view=icon,poster +0xdb.org/?find=title:foo,director:!bar&sort=country,-year&view=icon,poster // do we need this for search engines? +``` \ No newline at end of file diff --git a/RenamingFiles.md b/RenamingFiles.md new file mode 100644 index 0000000..1dd5046 --- /dev/null +++ b/RenamingFiles.md @@ -0,0 +1,45 @@ +## Semantic Meaning of Files + +Each movie has a number of meta files, each of which has a number of file instances (paths in archives). + +A meta file can have the following meaning: + +- Movie File (avi/mkv/...) + +- Movie File (avi/mkv/...), Part X + +- Subtitle File (srt/sub/idx/rar) + +- Subtitle File (srt/sub/idx/rar), Part X + +- Other File (mp3/txt/...) + +- Other File (mp3/txt/...), Part X + +## Potential Errors and Warnings + +Before files in an archive can be automatically renamed, we have to check for the following conditions: + +Errors: + +- "Concurrent Files" (multiple files with the same meaning and extension) + +Warnings: + +- "Multiple Files" (movie files with same meaning, but different extension) + +- "Incorrect Subtitles" (there are subtitles, but not every movie file has subtitles, or not every subtitle file has a movie file) + +- "Incomplete Subtitles" (idx without sub, or sub without idx) + +- "Multiple Versions" (both non-part and part files) + +- "Multiple Names" (different strings in Title.Version Name.Part X.ext) + +- "Incorrect Parts" (parts are not 1, 2, 3 ... n) + +## Renaming Files + +Once the errors have been resolved, files can be renamed to: + +L/Lastname, Firstname/The Title (Year)/The Title[.Version Name][.Part X].ext \ No newline at end of file diff --git a/UseCases.md b/UseCases.md new file mode 100644 index 0000000..2278d4a --- /dev/null +++ b/UseCases.md @@ -0,0 +1,60 @@ +## Use Cases +[Documents, pandora*, 0xdb2*)]TOC(heading=Design + +### Visitor + * search for a specific video + * browse videos + * embed clip on own page + * download full video + * download selected clip from video + +### registered User + * add annotation + * manage videos in lists + * use browser extension to manage local videos + +### Contributor(is this same as user?) + + * create annotations before upload + * upload video and annotations + * add new annotations + * add geo data to location names + * manage videos in lists + * add keywords and categories intelligently (?) + * get some data about downloads, views of their video. (?) + + +### VIP + + * see more info about videos + +### Admin + + * change data for users, videos, layers + * statistics + * remove video + * manipulate other information in terminal + * run scripts to update information on all videos + * mange featured lists + * manage subdomains + * change ownership for all layers of one video + * change ownership for all layers of one video belonging to user a to user b + +### API/Bots/Embedding + * embed clips from videos on other pages + * embed maps with clips on other pages + * use api to build custom interfaces + * make sure POST requests are Cross-site scripting (XSS) save + * implement something like JSONP so that Api works when user can only use JS. (?) + * Allow user to select in-point, out-point and tracks to display, and generate an or