From 1bd6615f1698cf712587f2b8992cc697113cc72b Mon Sep 17 00:00:00 2001 From: j <0x006A@0x2620.org> Date: Mon, 12 Oct 2009 17:18:59 +0200 Subject: [PATCH] back to oxlib, package_dir does not work with pip/python setup.py develop --- {web => oxweb}/__init__.py | 0 {web => oxweb}/allmovie.py | 4 ++-- {web => oxweb}/auth.py | 0 {web => oxweb}/criterion.py | 10 +++++----- {web => oxweb}/dailymotion.py | 2 +- {web => oxweb}/epguides.py | 4 ++-- {web => oxweb}/google.py | 10 +++++----- {web => oxweb}/imdb.py | 24 ++++++++++++------------ {web => oxweb}/impawards.py | 6 +++--- {web => oxweb}/itunes.py | 8 ++++---- {web => oxweb}/karagarga.py | 6 +++--- {web => oxweb}/lyricsfly.py | 6 +++--- {web => oxweb}/metacritic.py | 4 ++-- {web => oxweb}/mininova.py | 10 +++++----- {web => oxweb}/movieposterdb.py | 4 ++-- {web => oxweb}/opensubtitles.py | 10 +++++----- {web => oxweb}/oxdb.py | 2 +- {web => oxweb}/piratecinema.py | 4 ++-- {web => oxweb}/rottentomatoes.py | 4 ++-- {web => oxweb}/spiegel.py | 22 +++++++++++----------- {web => oxweb}/thepiratebay.py | 10 +++++----- {web => oxweb}/torrent.py | 2 +- {web => oxweb}/tv.py | 4 ++-- {web => oxweb}/wikipedia.py | 4 ++-- {web => oxweb}/youtube.py | 4 ++-- requirements.txt | 2 +- setup.py | 5 ++--- 27 files changed, 85 insertions(+), 86 deletions(-) rename {web => oxweb}/__init__.py (100%) rename {web => oxweb}/allmovie.py (97%) rename {web => oxweb}/auth.py (100%) rename {web => oxweb}/criterion.py (94%) rename {web => oxweb}/dailymotion.py (96%) rename {web => oxweb}/epguides.py (95%) rename {web => oxweb}/google.py (82%) rename {web => oxweb}/imdb.py (97%) rename {web => oxweb}/impawards.py (96%) rename {web => oxweb}/itunes.py (98%) rename {web => oxweb}/karagarga.py (98%) rename {web => oxweb}/lyricsfly.py (87%) rename {web => oxweb}/metacritic.py (93%) rename {web => oxweb}/mininova.py (93%) rename {web => oxweb}/movieposterdb.py (95%) rename {web => oxweb}/opensubtitles.py (84%) rename {web => oxweb}/oxdb.py (90%) rename {web => oxweb}/piratecinema.py (80%) rename {web => oxweb}/rottentomatoes.py (90%) rename {web => oxweb}/spiegel.py (96%) rename {web => oxweb}/thepiratebay.py (93%) rename {web => oxweb}/torrent.py (98%) rename {web => oxweb}/tv.py (93%) rename {web => oxweb}/wikipedia.py (97%) rename {web => oxweb}/youtube.py (97%) diff --git a/web/__init__.py b/oxweb/__init__.py similarity index 100% rename from web/__init__.py rename to oxweb/__init__.py diff --git a/web/allmovie.py b/oxweb/allmovie.py similarity index 97% rename from web/allmovie.py rename to oxweb/allmovie.py index b189645..9e06ed4 100644 --- a/web/allmovie.py +++ b/oxweb/allmovie.py @@ -3,8 +3,8 @@ import re import time -from ox import stripTags, findRe -from ox.cache import readUrlUnicode +from oxlib import stripTags, findRe +from oxlib.cache import readUrlUnicode def getId(url): diff --git a/web/auth.py b/oxweb/auth.py similarity index 100% rename from web/auth.py rename to oxweb/auth.py diff --git a/web/criterion.py b/oxweb/criterion.py similarity index 94% rename from web/criterion.py rename to oxweb/criterion.py index c204360..256357a 100644 --- a/web/criterion.py +++ b/oxweb/criterion.py @@ -2,10 +2,10 @@ # vi:si:et:sw=4:sts=4:ts=4 import re -import ox.cache -from ox.cache import readUrlUnicode -from ox.html import stripTags -from ox.text import findRe, removeSpecialCharacters +import oxlib.cache +from oxlib.cache import readUrlUnicode +from oxlib.html import stripTags +from oxlib.text import findRe, removeSpecialCharacters import imdb @@ -32,7 +32,7 @@ def getData(id): try: html = readUrlUnicode(data["url"]) except: - html = ox.cache.getUrl(data["url"]) + html = oxlib.cache.getUrl(data["url"]) data["number"] = findRe(html, "

(.*?)

") data["title"] = findRe(html, "

(.*?)

") data["director"] = findRe(html, "

(.*?)

") diff --git a/web/dailymotion.py b/oxweb/dailymotion.py similarity index 96% rename from web/dailymotion.py rename to oxweb/dailymotion.py index 63e09dc..0ad729f 100644 --- a/web/dailymotion.py +++ b/oxweb/dailymotion.py @@ -2,7 +2,7 @@ # vi:si:et:sw=4:sts=4:ts=4 import re from urllib import unquote -from ox.cache import readUrl +from oxlib.cache import readUrl def getVideoUrl(url): diff --git a/web/epguides.py b/oxweb/epguides.py similarity index 95% rename from web/epguides.py rename to oxweb/epguides.py index d4ad1aa..8d68183 100644 --- a/web/epguides.py +++ b/oxweb/epguides.py @@ -3,8 +3,8 @@ import re import time -from ox import stripTags, findRe -from ox.cache import readUrlUnicode +from oxlib import stripTags, findRe +from oxlib.cache import readUrlUnicode import google diff --git a/web/google.py b/oxweb/google.py similarity index 82% rename from web/google.py rename to oxweb/google.py index 9c61b19..00028df 100644 --- a/web/google.py +++ b/oxweb/google.py @@ -10,8 +10,8 @@ import Queue import simplejson -import ox -from ox import stripTags +import oxlib +from oxlib import stripTags ''' @@ -30,8 +30,8 @@ FIXME: how search depper than first page? DEFAULT_MAX_RESULTS = 10 DEFAULT_TIMEOUT = 24*60*60 -def readUrl(url, data=None, headers=ox.net.DEFAULT_HEADERS, timeout=DEFAULT_TIMEOUT): - return ox.cache.readUrl(url, data, headers, timeout) +def readUrl(url, data=None, headers=oxlib.net.DEFAULT_HEADERS, timeout=DEFAULT_TIMEOUT): + return oxlib.cache.readUrl(url, data, headers, timeout) def quote_plus(s): return urllib.quote_plus(s.encode('utf-8')) @@ -52,6 +52,6 @@ def find(query, max_results=DEFAULT_MAX_RESULTS, timeout=DEFAULT_TIMEOUT): def _find(query): url = 'http://ajax.googleapis.com/ajax/services/search/web?v=1.0&q=%s' % quote_plus(query) - results = simplejson.loads(ox.cache.readUrlUnicode(url))['responseData']['results'] + results = simplejson.loads(oxlib.cache.readUrlUnicode(url))['responseData']['results'] return results diff --git a/web/imdb.py b/oxweb/imdb.py similarity index 97% rename from web/imdb.py rename to oxweb/imdb.py index bd59e63..cab5ba1 100644 --- a/web/imdb.py +++ b/oxweb/imdb.py @@ -8,19 +8,19 @@ import time from BeautifulSoup import BeautifulSoup import chardet -import ox -from ox import stripTags, decodeHtml, findRe, findString -import ox.cache -from ox.normalize import normalizeTitle, normalizeImdbId -from ox import * +import oxlib +from oxlib import stripTags, decodeHtml, findRe, findString +import oxlib.cache +from oxlib.normalize import normalizeTitle, normalizeImdbId +from oxlib import * import google ''' never timeout imdb data, to update cache remove data from cache folder ''' -def readUrlUnicode(url, data=None, headers=ox.cache.DEFAULT_HEADERS, timeout=-1): - return ox.cache.readUrlUnicode(url, data, headers, timeout) +def readUrlUnicode(url, data=None, headers=oxlib.cache.DEFAULT_HEADERS, timeout=-1): + return oxlib.cache.readUrlUnicode(url, data, headers, timeout) ''' check if result is valid while updating @@ -28,8 +28,8 @@ def validate(result, header): return header['status'] == u'200' try: - d = ox.cache.readUrlUnicode(url, data, headers, timeout=0, valid=validate) -except ox.cache.InvalidResult, e: + d = oxlib.cache.readUrlUnicode(url, data, headers, timeout=0, valid=validate) +except oxlib.cache.InvalidResult, e: print e.headers ''' @@ -682,10 +682,10 @@ def guess(title, director=''): search = 'site:imdb.com "%s"' % title for (name, url, desc) in google.find(search, 2): if url.startswith('http://www.imdb.com/title/tt'): - return normalizeImdbId(int(ox.intValue(url))) + return normalizeImdbId(int(oxlib.intValue(url))) try: - req = urllib2.Request(imdb_url, None, ox.net.DEFAULT_HEADERS) + req = urllib2.Request(imdb_url, None, oxlib.net.DEFAULT_HEADERS) u = urllib2.urlopen(req) data = u.read() return_url = u.url @@ -700,7 +700,7 @@ def guess(title, director=''): return imdb_id imdb_url = 'http://www.imdb.com/find?q=%s;s=tt;site=aka' % quote(title.encode('utf-8')) - req = urllib2.Request(imdb_url, None, ox.net.DEFAULT_HEADERS) + req = urllib2.Request(imdb_url, None, oxlib.net.DEFAULT_HEADERS) u = urllib2.urlopen(req) data = u.read() return_url = u.url diff --git a/web/impawards.py b/oxweb/impawards.py similarity index 96% rename from web/impawards.py rename to oxweb/impawards.py index 7d027a4..7616d7e 100644 --- a/web/impawards.py +++ b/oxweb/impawards.py @@ -2,9 +2,9 @@ # encoding: utf-8 import re -from ox.cache import readUrlUnicode -from ox.html import stripTags -from ox.text import findRe +from oxlib.cache import readUrlUnicode +from oxlib.html import stripTags +from oxlib.text import findRe import imdb diff --git a/web/itunes.py b/oxweb/itunes.py similarity index 98% rename from web/itunes.py rename to oxweb/itunes.py index 5348e40..a9afd34 100644 --- a/web/itunes.py +++ b/oxweb/itunes.py @@ -3,10 +3,10 @@ import re import urllib -from ox.cache import readUrl -from ox.html import decodeHtml, stripTags -from ox.text import findRe -from ox.text import findString +from oxlib.cache import readUrl +from oxlib.html import decodeHtml, stripTags +from oxlib.text import findRe +from oxlib.text import findString # to sniff itunes traffic, use something like diff --git a/web/karagarga.py b/oxweb/karagarga.py similarity index 98% rename from web/karagarga.py rename to oxweb/karagarga.py index dfb667b..8f49656 100644 --- a/web/karagarga.py +++ b/oxweb/karagarga.py @@ -1,7 +1,7 @@ import re -from ox import cache -from ox.html import stripTags -from ox.text import findRe +from oxlib import cache +from oxlib.html import stripTags +from oxlib.text import findRe import auth diff --git a/web/lyricsfly.py b/oxweb/lyricsfly.py similarity index 87% rename from web/lyricsfly.py rename to oxweb/lyricsfly.py index 6a9b58a..422fc00 100644 --- a/web/lyricsfly.py +++ b/oxweb/lyricsfly.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -from ox.cache import readUrl -from ox.html import decodeHtml -from ox.text import findRe +from oxlib.cache import readUrl +from oxlib.html import decodeHtml +from oxlib.text import findRe def getLyrics(title, artist): diff --git a/web/metacritic.py b/oxweb/metacritic.py similarity index 93% rename from web/metacritic.py rename to oxweb/metacritic.py index 34e20a3..5b9edd6 100644 --- a/web/metacritic.py +++ b/oxweb/metacritic.py @@ -3,8 +3,8 @@ import re from urllib import quote -from ox.cache import readUrl, readUrlUnicode -from ox import findRe, decodeHtml, stripTags +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findRe, decodeHtml, stripTags def getMetacriticShowUrl(title): diff --git a/web/mininova.py b/oxweb/mininova.py similarity index 93% rename from web/mininova.py rename to oxweb/mininova.py index bfaa776..962b12f 100644 --- a/web/mininova.py +++ b/oxweb/mininova.py @@ -5,10 +5,10 @@ import re import socket from urllib import quote -from ox.cache import readUrl, readUrlUnicode -from ox import findRe, cache, stripTags, decodeHtml, getTorrentInfo, intValue, normalizeNewlines -from ox.normalize import normalizeImdbId -import ox +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findRe, cache, stripTags, decodeHtml, getTorrentInfo, intValue, normalizeNewlines +from oxlib.normalize import normalizeImdbId +import oxlib from torrent import Torrent @@ -55,7 +55,7 @@ def getId(mininovaId): def exists(mininovaId): mininovaId = getId(mininovaId) - data = ox.net.readUrl("http://www.mininova.org/tor/%s" % mininovaId) + data = oxlib.net.readUrl("http://www.mininova.org/tor/%s" % mininovaId) if not data or 'Torrent not found...' in data: return False if 'tracker of this torrent requires registration.' in data: diff --git a/web/movieposterdb.py b/oxweb/movieposterdb.py similarity index 95% rename from web/movieposterdb.py rename to oxweb/movieposterdb.py index 0068123..c64d211 100644 --- a/web/movieposterdb.py +++ b/oxweb/movieposterdb.py @@ -3,8 +3,8 @@ import re -from ox.cache import readUrlUnicode -from ox import findRe +from oxlib.cache import readUrlUnicode +from oxlib import findRe def getData(id): ''' diff --git a/web/opensubtitles.py b/oxweb/opensubtitles.py similarity index 84% rename from web/opensubtitles.py rename to oxweb/opensubtitles.py index 3d872ba..10af614 100644 --- a/web/opensubtitles.py +++ b/oxweb/opensubtitles.py @@ -3,9 +3,9 @@ import re import feedparser -from ox.cache import readUrl, readUrlUnicode -import ox -from ox import langCode2To3, langTo3Code +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findRe, stripTags +from oxlib import langCode2To3, langTo3Code def findSubtitlesByImdb(imdb, parts = 1, language = "eng"): if len(language) == 2: @@ -26,7 +26,7 @@ def findSubtitlesByImdb(imdb, parts = 1, language = "eng"): if opensubtitleId: opensubtitleId = opensubtitleId[0] else: - opensubtitleId = ox.findRe(data, '/en/subtitles/(.*?)/') + opensubtitleId = findRe(data, '/en/subtitles/(.*?)/') return opensubtitleId def downloadSubtitleById(opensubtitle_id): @@ -34,7 +34,7 @@ def downloadSubtitleById(opensubtitle_id): data = readUrl('http://www.opensubtitles.org/en/subtitles/%s' % opensubtitle_id) reg_exp = 'href="(/en/download/file/.*?)">(.*?)' for f in re.compile(reg_exp, re.DOTALL).findall(data): - name = ox.stripTags(f[1]).split('\n')[0] + name = stripTags(f[1]).split('\n')[0] url = "http://www.opensubtitles.com%s" % f[0] srts[name] = readUrlUnicode(url) return srts diff --git a/web/oxdb.py b/oxweb/oxdb.py similarity index 90% rename from web/oxdb.py rename to oxweb/oxdb.py index 7c15df0..9667f4e 100644 --- a/web/oxdb.py +++ b/oxweb/oxdb.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -import ox.cache +import oxlib.cache def getPosterUrl(id): url = "http://0xdb.org/%s/poster.0xdb.jpg" % id diff --git a/web/piratecinema.py b/oxweb/piratecinema.py similarity index 80% rename from web/piratecinema.py rename to oxweb/piratecinema.py index e429c7b..5a8fe31 100644 --- a/web/piratecinema.py +++ b/oxweb/piratecinema.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -import ox.cache -from ox.cache import exists +import oxlib.cache +from oxlib.cache import exists def getPosterUrl(id): diff --git a/web/rottentomatoes.py b/oxweb/rottentomatoes.py similarity index 90% rename from web/rottentomatoes.py rename to oxweb/rottentomatoes.py index 1a8106a..2eaf2dc 100644 --- a/web/rottentomatoes.py +++ b/oxweb/rottentomatoes.py @@ -2,8 +2,8 @@ # vi:si:et:sw=4:sts=4:ts=4 import re -from ox.cache import getHeaders, readUrl, readUrlUnicode -from ox import findRe, stripTags +from oxlib.cache import getHeaders, readUrl, readUrlUnicode +from oxlib import findRe, stripTags def readUrlByImdb(imdb): diff --git a/web/spiegel.py b/oxweb/spiegel.py similarity index 96% rename from web/spiegel.py rename to oxweb/spiegel.py index abfe189..403b9f6 100644 --- a/web/spiegel.py +++ b/oxweb/spiegel.py @@ -6,9 +6,9 @@ import time from BeautifulSoup import BeautifulSoup -import ox.cache -from ox.html import decodeHtml, stripTags -import ox.net +import oxlib.cache +from oxlib.html import decodeHtml, stripTags +import oxlib.net def getNews(year, month, day): @@ -23,9 +23,9 @@ def getNews(year, month, day): for section in sections: url = 'http://www.spiegel.de/%s/0,1518,archiv-%d-%03d,00.html' % (section, year, day) if date == time.strftime('%d.%m.%Y', time.localtime()): - html = ox.net.readUrl(url) + html = oxlib.net.readUrl(url) else: - html = ox.cache.readUrl(url) + html = oxlib.cache.readUrl(url) for item in re.compile('
(.*?)
', re.DOTALL).findall(item)[0]).strip() try: @@ -102,11 +102,11 @@ def formatSubsection(string): def getIssue(year, week): coverUrl = 'http://www.spiegel.de/static/epaper/SP/%d/%d/ROSPANZ%d%03d0001-312.jpg' % (year, week, year, week) - if not ox.net.exists(coverUrl): + if not oxlib.net.exists(coverUrl): return None url = 'http://service.spiegel.de/digas/servlet/epaper?Q=SP&JG=%d&AG=%d&SE=1&AN=INHALT' % (year, week) contents = [] - soup = BeautifulSoup(ox.cache.readUrl(url)) + soup = BeautifulSoup(oxlib.cache.readUrl(url)) for item in soup('a', {'href': re.compile('http://service.spiegel.de/digas/servlet/epaper\?Q=SP&JG=')}): item = str(item) page = int(re.compile('&SE=(.*?)"').findall(item)[0]) @@ -116,7 +116,7 @@ def getIssue(year, week): pages = page + 2 for page in range(1, pages + 10): url = 'http://www.spiegel.de/static/epaper/SP/%d/%d/ROSPANZ%d%03d%04d-205.jpg' % (year, week, year, week, page) - if ox.cache.exists(url): + if oxlib.cache.exists(url): pageUrl[page] = url else: pageUrl[page] = '' @@ -164,7 +164,7 @@ def archiveIssues(): f.close() filename = '%s/Der Spiegel %d %02d.jpg' % (dirname, y, w) if not os.path.exists(filename): - data = ox.cache.readUrl(issue['coverUrl']) + data = oxlib.cache.readUrl(issue['coverUrl']) f = open(filename, 'w') f.write(data) f.close() @@ -173,7 +173,7 @@ def archiveIssues(): if url: filename = '%s/Der Spiegel %d %02d %03d.jpg' % (dirname, y, w, page) if not os.path.exists(filename): - data = ox.cache.readUrl(url) + data = oxlib.cache.readUrl(url) f = open(filename, 'w') f.write(data) f.close() @@ -244,7 +244,7 @@ def archiveNews(): f.close() filename = dirname + '/' + new['imageUrl'].split('/')[-1] if not os.path.exists(filename): - data = ox.cache.readUrl(new['imageUrl']) + data = oxlib.cache.readUrl(new['imageUrl']) f = open(filename, 'w') f.write(data) f.close() diff --git a/web/thepiratebay.py b/oxweb/thepiratebay.py similarity index 93% rename from web/thepiratebay.py rename to oxweb/thepiratebay.py index 4202a4d..ef55f03 100644 --- a/web/thepiratebay.py +++ b/oxweb/thepiratebay.py @@ -6,10 +6,10 @@ import socket from urllib import quote, urlencode from urllib2 import URLError -from ox.cache import readUrl, readUrlUnicode -from ox import findRe, cache, stripTags, decodeHtml, getTorrentInfo, normalizeNewlines -from ox.normalize import normalizeImdbId -import ox +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findRe, cache, stripTags, decodeHtml, getTorrentInfo, normalizeNewlines +from oxlib.normalize import normalizeImdbId +import oxlib from torrent import Torrent @@ -67,7 +67,7 @@ def getId(piratebayId): def exists(piratebayId): piratebayId = getId(piratebayId) - return ox.net.exists("http://thepiratebay.org/torrent/%s" % piratebayId) + return oxlib.net.exists("http://thepiratebay.org/torrent/%s" % piratebayId) def getData(piratebayId): _key_map = { diff --git a/web/torrent.py b/oxweb/torrent.py similarity index 98% rename from web/torrent.py rename to oxweb/torrent.py index 68cd274..9188716 100644 --- a/web/torrent.py +++ b/oxweb/torrent.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # vi:si:et:sw=4:sts=4:ts=4 -from ox import intValue +from oxlib import intValue class Torrent(dict): diff --git a/web/tv.py b/oxweb/tv.py similarity index 93% rename from web/tv.py rename to oxweb/tv.py index 3808bbd..201b418 100644 --- a/web/tv.py +++ b/oxweb/tv.py @@ -3,8 +3,8 @@ import re import time -from ox import stripTags, findRe -from ox.cache import readUrlUnicode +from oxlib import stripTags, findRe +from oxlib.cache import readUrlUnicode def getEpisodeData(url): diff --git a/web/wikipedia.py b/oxweb/wikipedia.py similarity index 97% rename from web/wikipedia.py rename to oxweb/wikipedia.py index 3d99688..a0f4132 100644 --- a/web/wikipedia.py +++ b/oxweb/wikipedia.py @@ -3,8 +3,8 @@ from urllib import urlencode import simplejson -from ox.cache import readUrl, readUrlUnicode -from ox import findRe, decodeHtml +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findRe, decodeHtml def getId(url): diff --git a/web/youtube.py b/oxweb/youtube.py similarity index 97% rename from web/youtube.py rename to oxweb/youtube.py index cb9ca53..cf640c9 100644 --- a/web/youtube.py +++ b/oxweb/youtube.py @@ -6,8 +6,8 @@ import xml.etree.ElementTree as ET import re import feedparser -from ox.cache import readUrl, readUrlUnicode -from ox import findString, findRe +from oxlib.cache import readUrl, readUrlUnicode +from oxlib import findString, findRe def getVideoKey(youtubeId): diff --git a/requirements.txt b/requirements.txt index 3fec7b5..67f19ac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -ox +oxlib diff --git a/setup.py b/setup.py index 7435af9..f4bc37b 100644 --- a/setup.py +++ b/setup.py @@ -16,11 +16,10 @@ setup( description="collection of scrapers for the web", author="0x", author_email="code@0xdb.org", - url="http://code.0xdb.org/oxweb", + url="http://code.0xdb.org/python-oxweb", download_url="http://code.0xdb.org/oxweb/download", license="GPLv3", - package_dir = {'ox.web': 'web'}, - packages=['ox.web'], + packages=['oxweb'], keywords = [ ], classifiers = [