add ipv4 map example

This commit is contained in:
rolux 2012-06-21 23:17:02 +02:00
parent 21fb4aecc0
commit d5e2fcc1c2
15 changed files with 1488 additions and 0 deletions

View file

@ -0,0 +1,176 @@
#map {
position: absolute;
width: 100%;
height: 100%;
background-color: rgb(64, 64, 64);
}
#tiles {
position: absolute;
background-color: rgb(224, 224, 224);
background-image:
-moz-linear-gradient(
45deg, rgb(160, 160, 160) 25%, transparent 25%,
transparent 75%, rgb(160, 160, 160) 75%, rgb(160, 160, 160)
),
-moz-linear-gradient(
45deg, rgb(160, 160, 160) 25%, transparent 25%,
transparent 75%, rgb(160, 160, 160) 75%, rgb(160, 160, 160)
);
background-image:
-webkit-linear-gradient(
45deg, rgb(160, 160, 160) 25%, transparent 25%,
transparent 75%, rgb(160, 160, 160) 75%, rgb(160, 160, 160)
),
-webkit-linear-gradient(
45deg, rgb(160, 160, 160) 25%, transparent 25%,
transparent 75%, rgb(160, 160, 160) 75%, rgb(160, 160, 160)
);
background-size: 32px 32px;
background-position: 0 0, 16px 16px;
box-shadow: 0 0 16px black;
cursor: move;
}
#overlay {
position: absolute;
opacity: 0;
z-index: 1;
}
.tile {
position: absolute;
width: 256px;
height: 256px;
}
.interface {
position: absolute;
box-shadow: 0 0 16px rgb(0, 0, 0);
z-index: 3;
}
.interface .interface {
z-index: 5;
}
#about {
left: 16px;
top: 16px;
}
#find {
right: 16px;
top: 16px;
border-radius: 8px;
}
#toggle {
right: 16px;
bottom: 16px;
}
#zoom {
left: 16px;
bottom: 16px;
border-radius: 8px;
}
#world {
right: 16px;
bottom: 48px;
width: 256px;
height: 256px;
box-shadow: 0 0 16px rgb(0, 0, 0);
opacity: 0;
overflow: hidden;
}
div.marker {
position: absolute;
width: 16px;
height: 16px;
padding: 8px;
border-radius: 16px;
margin: -16px 0 0 -16px;
background-image: -moz-linear-gradient(top, rgba(255, 255, 255, 0.75), rgba(192, 192, 192, 0.75));
background-image: -webkit-linear-gradient(top, rgba(255, 255, 255, 0.75), rgba(192, 192, 192, 0.75));
box-shadow: 0 0 2px rgb(0, 0, 0), 0 0 2px rgb(0, 0, 0) inset;
cursor: pointer;
z-index: 2;
}
div.marker.selected {
padding: 7px;
border: 1px solid rgb(64, 64, 64);
box-shadow: 0 0 4px rgb(0, 0, 0), 0 0 4px rgb(0, 0, 0) inset;
background-image: -moz-linear-gradient(top, rgba(192, 192, 192, 0.75), rgba(192, 192, 192, 0.75));
background-image: -webkit-linear-gradient(top, rgba(192, 192, 192, 0.75), rgba(192, 192, 192, 0.75));
}
#world div.marker {
width: 16px;
height: 16px;
padding: 0;
border-radius: 16px;
margin: -8px 0 0 -8px;
z-index: 4;
}
img.marker {
width: 16px;
height: 16px;
}
.textTitle {
float: left;
width: 384px;
margin: 16px 0 8px 144px;
font-weight: bold;
}
.textTitle:first-child {
margin-top: 0;
}
.textKey {
float: left;
width: 128px;
margin-right: 16px;
text-align: right;
}
.textValue {
float: left;
width: 384px;
}
#regions {
position: absolute;
}
.region {
position: absolute;
background-color: rgba(0, 0, 0, 0.5);
}
.region.ui {
box-shadow: 0 0 2px rgb(0, 0, 0), 0 0 2px rgb(0, 0, 0) inset;
}
.region#center {
left: 256px;
top: 256px;
box-shadow: 0 0 2px rgb(0, 0, 0), 0 0 2px rgb(0, 0, 0) inset;
background-color: transparent;
}
.region#left {
left: 0;
top: 0;
bottom: 0;
width: 256px;
}
.region#right {
right: 0;
top: 0;
bottom: 0;
width: 256px;
}
.region#top {
top: 0;
left: 256px;
height: 256px;
}
.region#bottom {
bottom: 0;
left: 256px;
height: 256px;
}
.OxTooltip {
box-shadow: 0 0 16px rgb(0, 0, 0);
}

View file

@ -0,0 +1,15 @@
<p>This is a world map of the Internet. It shows how the IPv4 address space is distributed among the world's countries (hint: unevenly!). If you zoom all the way out, you can see the entire Internet. If you zoom all the way in, each pixel is one unique IP address.</p>
<p>The initial idea was to visualize the data provided by <a href="http://maxmind.com">MaxMind</a>'s free <a href="http://www.maxmind.com/app/geolitecity">GeoLite City</a> database. The algorithm that translates IP addresses to map coordinates was inspired by the <a href="http://xkcd.com/195/" target="_blank">xkcd</a> comic you see on the right. It uses a fractal mapping, so that consecutive addresses form compact and contiguous regions.</p>
<p>The map tiles are rendered using a recursive function. When passing a square region of the map to this function, it calls itself for each quarter of the square, and so on. If all four calls return the same country, it returns that country. Otherwise, it returns an image, rendered by combining any images that were returned with the flag of each country that was returned. This way, large blocks of IP addresses appear as large blocks on the map.</p>
<p>The mapping algorithm becomes quite simple once you realize that there are only four possible path segments &mdash; their shapes resemble the letters A, C, D and U &mdash; and that they decompose into each other. The U segment, for example, becomes D-U-U-C. The image on the right shows the path from 0.X.X.X to 255.X.X.X.</p>
<p>Each tile has a size of 256 by 256 pixels, and its file name is the range of IP addresses it depicts. For example, at the highest zoom level, the the top left corner tile is "0/0.0.0.0-0.0.255.255.png". This naming scheme is pretty, but combined with the fractal projection, it makes translating screen coordinates to tile names somewhat more difficult than in most other map applications.</p>
<p>Another challenge is the fact that nation states, since they were invented long before the Internet, are represented by flags and not by icons. These flags tend to have weird aspect ratios and don't work well on a map that is essentially a square of squares. One could just crop or resize them, but in most cases, the results look surprisingly stupid.</p>
<p>Luckily, <a href="http://oxjs.org" target="blank">OxJS</a> solves this problem. Its Geo module comes with a set of square flag icons. They're generated (almost) automatically, by fetching SVGs from wikipedia and applying some more or less regular transformations. The French Tricolore is easy. The Union Jack is harder. And the flag of Nepal is <i>really</i> hard. (But the hardest part was figuring out how to render SVGs as PNGs, in python, on Mac OS X, without installing obscure libraries that have insane dependencies. It turned out to be a one-liner: <b><code>os.system('qlmanage -t -s %d -o %s %s' % (size, path, svg))</code></b>. Making these icons was definitely more time-consuming than making this map.</p>
<p>Generating the images took <a href="txt/ipv4map.py.txt" target="_blank">200 lines of Python</a>. The map client is <a href="txt/ipv4map.js.txt" target="_blank">1,000 lines of JavaScript</a>. And the server backend is <a href="txt/ipv4map.php.txt" target="_blank">5 lines of PHP</a>. If you find bugs, or spot anything that should be done more elegantly, your <a href="http://0x2620.org/0x0073/">feedback</a> is appreciated.</p>

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,50 @@
[
{"id": "interesting", "title": "Interesting", "items": [
{"id": "122.152.72.0", "title": "Asia"},
{"id": "203.32.1.1", "title": "Australia"},
{"id": "91.221.91.221", "title": "Balkans"},
{"id": "242.170.138.80", "title": "Iceland"},
{"id": "202.224.1.1", "title": "Japan"},
{"id": "204.106.204.106", "title": "Midwest"},
{"id": "84.22.111.22", "title": "South Pole"},
{"id": "188.188.188.188", "title": "Vennbahn"}
]},
{"id": "popular", "title": "Popular", "items": [
{"id": "adobe.com", "title": "Adobe"},
{"id": "amazon.com", "title": "Amazon"},
{"id": "apple.com", "title": "Apple"},
{"id": "ebay.com", "title": "eBay"},
{"id": "facebook.com", "title": "Facebook"},
{"id": "flickr.com", "title": "Flickr"},
{"id": "google.com", "title": "Google"},
{"id": "microsoft.com", "title": "Microsoft"},
{"id": "thepiratebay.se", "title": "The Pirate Bay"},
{"id": "reddit.com", "title": "Reddit"},
{"id": "sony.com", "title": "Sony"},
{"id": "tumblr.com", "title": "Tumblr"},
{"id": "twitter.com", "title": "Twitter"},
{"id": "wikipedia.org", "title": "Wikipedia"},
{"id": "yahoo.com", "title": "Yahoo"},
{"id": "youtube.com", "title": "Youtube"}
]},
{"id": "news", "title": "News", "items": [
{"id": "cnn.com", "title": "CNN"},
{"id": "elpais.com", "title": "El Pais"},
{"id": "guardian.co.uk", "title": "Guardian"},
{"id": "lemonde.fr", "title": "Le Monde"},
{"id": "nytimes.com", "title": "NY Times"},
{"id": "rollingstone.com", "title": "Rolling Stone"},
{"id": "salon.com", "title": "Salon"},
{"id": "spiegel.de", "title": "Spiegel"}
]},
{"id": "technology", "title": "Technology", "items": [
{"id": "arstechnica.com", "title": "Ars Technica"},
{"id": "eff.org", "title": "EFF"},
{"id": "github.com", "title": "GitHub"},
{"id": "mozilla.org", "title": "Mozilla"},
{"id": "news.ycombinator.com", "title": "Hacker News"},
{"id": "slashdot.org", "title": "Slashdot"},
{"id": "stackoverflow.com", "title": "Stack Overflow"},
{"id": "wired.com", "title": "Wired"}
]}
]

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 490 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 396 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 699 KiB

View file

@ -0,0 +1,14 @@
<?php
// ipv4map.php - no copyright 2011 0x2620.org - public domain
$data = array();
$data['host'] = $_GET['host'] ? $_GET['host']
: gethostbyaddr($_GET['ip'] ? $_GET['ip'] : $_SERVER['REMOTE_ADDR']);
$data['ip'] = $_GET['ip'] ? $_GET['ip']
: $_GET['host'] ? gethostbyname($data['host']) : $_SERVER['REMOTE_ADDR'];
header('Content-Type: text/javascript');
echo $_GET['callback'] . '(' . json_encode($data) . ')';
?>

View file

@ -0,0 +1,191 @@
# ipv4map.py - no copyright 2011 0x2620.org - public domain
from __future__ import division
import Image
import ImageDraw
import math
import os
import ox
import pygeoip
# fixme: pow(a, b) -> a ** b
geoip = pygeoip.GeoIP('../dat/GeoLiteCity.dat', pygeoip.MEMORY_CACHE)
path = '../../../../oxjs.org/source/Ox.Geo/png/flags/'
projection = {
'U': [[0, 2, 3, 1], 'DUUC'], # Square 0 1 Segment U = 0 3 = 0 1 E F = D C
'D': [[0, 1, 3, 2], 'UDDA'], # 2 3 1 2 3 2 D C U U
'C': [[3, 2, 0, 1], 'ACCU'], # 4 7 8 B
'A': [[3, 1, 0, 2], 'CAAD'] # 5 6 9 A
}
def get_country_code_by_ip(ip):
replace = {'A1': 'NTHH', 'A2': 'NTHH', 'AN': 'ANHH', 'AP': 'FM', 'O1': 'NTHH'}
data = geoip.record_by_addr(ip) if not ip.startswith('0.') else None
country_code = data['country_code'] if data and 'country_code' in data else 'NTHH'
return replace[country_code] if country_code in replace else country_code
def get_flag_image(county_code, size):
flag_size = str(int(pow(4, math.ceil(math.log(size, 2) / 2))))
flag_image = Image.open(path + flag_size + '/' + county_code + '.png')
if size != flag_size:
flag_image = flag_image.resize((size, size), Image.ANTIALIAS)
return flag_image
def get_ip_by_n(n):
ip = []
for i in range(4):
ip.append(str(int(n / pow(256, 3 - i)) % 256))
return '.'.join(ip)
def get_n_by_ip(ip):
n = 0
for i, v in enumerate(ip.split('.')):
n += int(v) * pow(256, 3 - i)
return n
def get_tile_by_ip(ip, z):
n = get_n_by_ip(ip)
last_n = n + pow(4, 16 - z) - 1
last_ip = get_ip_by_n(last_n)
return '../png/tiles/%s/%s-%s.png' % (ip.split('.')[0], ip, last_ip)
def get_xy_by_ip(ip, z, w='U'):
x, y = 0, 0
n = int(get_n_by_ip(ip) / pow(4, 8 - z))
for i in range(8 + z):
p2 = pow(2, 7 + z - i)
p4 = pow(4, 7 + z - i)
q = int(n / p4)
xy = projection[w][0][q]
x += xy % 2 * p2
y += int(xy / 2) * p2
n -= q * p4
w = projection[w][1][q]
return [x, y]
def render_flags():
# renders an image of 256 flags
image = Image.new('RGB', (1024, 1024))
font = '../ttf/DejaVuSansMonoBold.ttf'
countries = ox.file.read_json('../../../../oxjs.org/oxjs/source/Ox.Geo/json/Ox.Geo.json')
countries = filter(lambda x: len(x['code']) == 2 and not x['code'] in ['UK', 'XK'], countries)
for i, country in enumerate(sorted(countries, key=lambda x: x['code'])):
flag = get_flag_image(country['code'], 64)
x, y = i % 16 * 64, int(i / 16) * 64
image.paste(flag, (x, y, x + 64, y + 64))
ox.image.drawText(image, (x + 5, y + 5), country['code'], font, 8, (64, 64, 64))
ox.image.drawText(image, (x + 4, y + 4), country['code'], font, 8, (192, 192, 192))
image.save('../png/flags.png')
def render_map():
# renders images of the full map
if not os.path.exists('../png/ipv4map16.png'):
mapfile = '../png/ipv4map16384.png'
mapimage = Image.new('RGB', (16384, 16384))
for a in range(0, 256, 16):
print '%d.0.0.0' % a
image = Image.open('../png/map/%d.0.0.0-%d.255.255.255.png' % (a, a + 15))
x, y = map(lambda x: int(x / 4096) * 4096, get_xy_by_ip('%d.0.0.0' % a, 6))
mapimage.paste(image.resize((4096, 4096), Image.ANTIALIAS), (x, y))
mapimage.save(mapfile)
for s in [4096, 1024, 16]:
mapimage.resize((s, s), Image.ANTIALIAS).save(mapfile.replace('16384', str(s)))
def render_square(values, maxlen, w):
# recursively renders a square region, given a list of country codes
length = len(values)
if length > 4:
square_length = int(length / 4)
square_values = map(lambda x: values[x * square_length:(x + 1) * square_length], range(4))
values = []
for i, v in enumerate(square_values):
values.append(render_square(v, maxlen, projection[w][1][i]))
equal = values[0] == values[1] == values[2] == values[3]
if length < maxlen and equal:
return values[0]
else:
order = projection[w][0]
image_size = int(math.sqrt(length) * 16)
flag_size = image_size if equal else int(image_size / 2)
image = Image.new('RGB', (image_size, image_size))
for i, value in enumerate([values[0]] if equal else values):
flag = get_flag_image(value, flag_size) if isinstance(value, str) else value
image.paste(flag, (order[i] % 2 * flag_size, int(order[i] / 2) * flag_size))
if length == pow(4, 4):
image.save('../png/_tmp/' + str(values) + '.png')
return image
def render_tile(ip, z):
# recursively renders the map tile for a given ip at a given zoom level
n = get_n_by_ip(ip)
file = get_tile_by_ip(ip, z)
if not os.path.exists(file):
tiles_n = map(lambda x: n + x * pow(4, 15 - z), range(4))
tiles_ip = map(lambda x: get_ip_by_n(x), tiles_n)
tiles = map(lambda x: render_tile(x, z + 1), tiles_ip)
image = Image.new('RGB', (256, 256))
x, y = map(lambda x: int(x / 256) * 256, get_xy_by_ip(ip, z))
for i in range(4):
x_, y_ = map(lambda x: int(x / 128) * 128, get_xy_by_ip(tiles_ip[i], z))
image.paste(tiles[i].resize((128, 128), Image.ANTIALIAS), (x_ - x, y_ - y))
image.save(file)
else:
image = Image.open(file)
return image
def render_tiles():
# renders all tiles at the maximum zoom level
w = ''
for v in projection['U'][1]:
w += projection[v][1]
file256 = '../png/ipv4map16384.png'
for a in range(256):
print '%d.0.0.0' % a
if a % 16 == 0:
values = []
file = '../png/map/%d.0.0.0-%d.255.255.255.png' % (a, a + 15)
if not os.path.exists(file):
for b in range(256):
for c in range(256):
values.append(get_country_code_by_ip('%d.%d.%d.0' % (a, b, c)))
if a % 16 == 15:
image = render_square(values, len(values), w[int(a / 16)]).save(file)
if a % 16 == 15:
image = Image.open(file)
for a_ in range(a - 15, a + 1):
for b in range(256):
file = '../png/tiles/%d/%d.%d.0.0-%d.%d.255.255.png' % (a_, a_, b, a_, b)
if not os.path.exists(file):
x, y = map(lambda x: int(x % 16384 / 256) * 256, get_xy_by_ip('%d.%d.0.0' % (a_, b), 8))
dirs = os.path.split(file)[0]
if dirs and not os.path.exists(dirs):
os.makedirs(dirs)
image.crop((x, y, x + 256, y + 256)).save(file)
def render_projection():
# renders an image of the map projection
image = Image.new('RGB', (1024, 1024))
draw = ImageDraw.Draw(image)
font = '../ttf/DejaVuSansCondensedBold.ttf'
for i in range(256):
x, y = map(lambda x: int(x / 64) * 64, get_xy_by_ip('%d.0.0.0' % i, 2))
rgb = ox.getRGB((i * 360 / 256, 1, 0.5))
draw.rectangle((x, y, x + 64, y + 64), fill=rgb)
ox.image.drawText(image, (x + 4, y + 4), str(i), font, 8, (0, 0, 0))
draw.rectangle((x + 30, y + 30, x + 34, y + 33), fill=0)
if i:
draw.line((x_ + 32, y_ + 32, x + 32, y + 32), fill=0, width=4)
x_, y_ = x, y
image.save('../png/projection.png')
if __name__ == '__main__':
# qlmanage -t -s 16384 -o . ../../../../oxjs.org/tools/geo/svg/icons/NTHH.svg
render_flags()
render_projection()
render_tiles()
render_tile('0.0.0.0', 0)
render_map()
# http://s3.amazonaws.com/alexa-static/top-1m.csv.zip