If it won't be simple, it simply won't be. [Hire me, source code] by Miki Tebeka, CEO, 353Solutions

Friday, June 12, 2009

A Twitter Trends / Google News meshup

This tried to automatically explain why things are trending on Twitter.

The Python Server (trends.py)


#!/usr/bin/env python
'''A twitter trends/google news mesh

loader_thread get news trends every 1min and set _TRENDS_HTML
The web server serves _TRENDS_HTML with is refresed via JavaScript every 30sec
'''

from urllib import urlopen, urlencode
import feedparser
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from os.path import dirname, join, splitext
import json
from threading import Thread
from time import sleep

def ascii_clean(text):
return text.encode("ascii", "ignore") # Pure ACII

def trend_news(trend):
query = {
"q" : ascii_clean(trend),
"output" : "rss"
}
url = "http://news.google.com/news?" + urlencode(query)
return feedparser.parse(url).entries

def current_trends():
url = "http://search.twitter.com/trends.json"
return json.load(urlopen(url))["trends"]

def news_html(news):
html = '<li><a href="%(link)s">%(title)s</a></li>'
chunks = map(lambda e: html % e, news)
return "\n".join(["<ul>"] + chunks + ["</ul>"])

def trend_html(trend):
thtml = '<a href="%(url)s">%(name)s' % trend
nhtml = news_html(trend_news(trend["name"]))
return '<tr><td>%s</td><td>%s</td><tr>' % (thtml, nhtml)

def table_html(trends):
return ("<table>" +
"<tr><th>Trend</th><th>Related News</th></tr>" +
"".join(map(trend_html, trends)) +
"</table>"
)

_TRENDS_HTML = ""
def loader_thread():
global _TRENDS_HTML

while 1:
_TRENDS_HTML = ascii_clean(table_html(current_trends()))
sleep(60)

def run_loader_thread():
t = Thread(target=loader_thread)
t.daemon = 1
t.start()

def trends_html():
while not _TRENDS_HTML:
sleep(0.1)

return _TRENDS_HTML

def index_html():
return open(join(dirname(__file__), "index.html")).read()

class RequestHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/":
self.wfile.write(index_html())
elif self.path.startswith("/trends"):
self.wfile.write(trends_html())
elif splitext(self.path)[1] in (".js", ".css"):
self.wfile.write(open(".%s" % self.path).read())
else:
self.send_error(404, "Not Found")

if __name__ == "__main__":
run_loader_thread()
server = HTTPServer(("", 8888), RequestHandler)
server.serve_forever()


The Auto-Refreshing Web Client (index.html)


<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>Trends</title>
<link rel="stylesheet" href="trends.css" />
<style>
body {
font-family: Serif, Helvetica;
}

table {
width: 100%;
border: 2px solid gray;
}

td {
vertical-align: top;
border-bottom: 1px solid black;
}

h1 {
text-align: center;
font-variant: small-caps;
}

a {
text-decoration: none;
color: black;
}

a:hover {
background: silver;
}

ul {
margin: 0px;
}
</style>
</head>
<body>
<h1>Trends</h1>
<div id="trends">
Loading ...
</div>
</body>
<script src="jquery.js"></script>
<script>
function load() {
$('#trends').load('/trends');
}
$(document).ready(function() {
load();
setInterval(load, 30 * 1000);
});
</script>
</html>

Wednesday, June 10, 2009

Show "digits only" version of phone number

#!/bin/bash
# Convert phone numbers to numeric (1-800-T-MOBILE -> 1-800-8-662453)

if [ $# -ne 1 ]; then
echo "usage: $(basename $0) PHONE-NUMBER"
exit 1
fi

echo $1 | \
tr '[:lower:]' '[:upper:]' | \
tr ABCDEFGHIJKLMNOPQRSTUVWXYZ 22233344455566677778889999

Saturday, June 06, 2009

JSON Pretiffier

#!/usr/bin/env python
'''JSON prettifier'''

def shift(array):
try:
return array.pop(0)
except IndexError:
return None

def fopen(name, mode, default):
if name in (None, "-"):
return default
return open(name, mode)

def main(argv=None):
import sys
from optparse import OptionParser

argv = argv or sys.argv

indent = 4
parser = OptionParser("%prog [INFILE [OUTFILE]]")
parser.add_option("-i", "--indent", help="indent size (%s)" % indent,
dest="indent", default=indent, type="int")
opts, args = parser.parse_args(argv[1:])
if len(args) not in (0, 1, 2):
parser.error("wrong number of arguments") # Will exit

try:
info = fopen(shift(args), "r", sys.stdin)
outfo = fopen(shift(args), "w", sys.stdout)
except IOError, e:
raise SystemExit("error: %s" % e)

import json
json.dump(json.load(info), outfo, indent=opts.indent)

if __name__ == "__main__":
main()


Update (Aug 14, 2009): Just found out about python -mjson.tool, oh well ...

Thursday, June 04, 2009

strftime for JavaScript

/* strftime for JavaScript 

   Field description (taken from http://tinyurl.com/65s2qw)

    %a  Locale’s abbreviated weekday name.   
    %A  Locale’s full weekday name.   
    %b  Locale’s abbreviated month name.   
    %B  Locale’s full month name.   
    %c  Locale’s appropriate date and time representation.   
    %d  Day of the month as a decimal number [01,31].   
    %H  Hour (24-hour clock) as a decimal number [00,23].   
    %I  Hour (12-hour clock) as a decimal number [01,12].   
    %j  Day of the year as a decimal number [001,366].   
    %m  Month as a decimal number [01,12].   
    %M  Minute as a decimal number [00,59].   
    %p  Locale’s equivalent of either AM or PM.
    %S  Second as a decimal number [00,61].
    %U  Week number of the year (Sunday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Sunday are considered to be in week 0.
    %w  Weekday as a decimal number [0(Sunday),6].   
    %W  Week number of the year (Monday as the first day of the week) as a
        decimal number [00,53]. All days in a new year preceding the first
        Monday are considered to be in week 0.
    %x  Locale’s appropriate date representation.   
    %X  Locale’s appropriate time representation.   
    %y  Year without century as a decimal number [00,99].   
    %Y  Year with century as a decimal number.   
    %Z  Time zone name (no characters if no time zone exists).   
    %%  A literal '%' character.
*/

var days = [ 
    'Sunday', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday',
    'Saturday' 
];

var months = [
    'January', 'February', 'March', 'April', 'May', 'June', 'July',
    'August', 'September', 'October', 'November', 'December'
];

function shortname(name) {
    return name.substr(0, 3);
}

function zeropad(n, size) {
    n = '' + n; /* Make sure it's a string */
    size = size || 2;
    while (n.length < size) {
        n = '0' + n;
    }
    return n;
}

function twelve(n) {
    return (n <= 12) ? n : 24 - n;
}

function strftime(format, date) {
    date = date || new Date();
    var fields = {
        a: shortname(days[date.getDay()]),
        A: days[date.getDay()],
        b: shortname(months[date.getMonth()]),
        B: months[date.getMonth()],
        c: date.toString(),
        d: zeropad(date.getDate()),
        H: zeropad(date.getHours()),
        I: zeropad(twelve(date.getHours())),
        /* FIXME: j: */
        m: zeropad(date.getMonth() + 1),
        M: zeropad(date.getMinutes()),
        p: (date.getHours() >= 12) ? 'PM' : 'AM',
        S: zeropad(date.getSeconds()),
        w: zeropad(date.getDay() + 1),
        /* FIXME: W: */
        x: date.toLocaleDateString(),
        X: date.toLocaleTimeString(),
        y: ('' + date.getFullYear()).substr(2, 4),
        Y: '' + date.getFullYear(),
        /* FIXME: Z: */
        '%' : '%'
    };

    var result = '', i = 0;
    while (i < format.length) {
        if (format[i] === '%') {
            result = result + fields[format[i + 1]];
            ++i;
        }
        else {
            result = result + format[i];
        }
        ++i;
    }
    return result;
}

Quick search using xapian "omega"

A little hack to search current directory using Xapian's omega.

#!/bin/bash

# Minimal search using Xpians Omega (http://xapian.org/docs/omega/overview.html)
# Use "omsearch -i" to index current directoy, then use "omsearch QUERY" to
# search


index() {
mkdir .omega
cat >> .omega/omega.conf << EOF
database_dir $PWD/.omega
template_dir /var/lib/xapian-omega/templates
log_dir /tmp
EOF
omindex --db .omega/default .
}

search() {
omega=/usr/lib/cgi-bin/omega/omega
config=$PWD/.omega/omega.conf
results=.omega/results.html
OMEGA_CONFIG_FILE=$config $omega "P=$1" "HITSPERPAGE=1000" > $results
egrep -o '<B><A HREF=".*"' $results | cut -d\" -f2 | \
xargs -i printf ".%s\n" {}

}

USAGE="usage: $(basename $0) [-i | QUERY]"

index=no
while getopts "ih" opt
do
case $opt in
h | H ) echo $USAGE; exit;;
i ) index=yes;;
* ) echo "error: unknown option" >&2; exit 1;;
esac
done

shift $(($OPTIND - 1))

if [ "$index" == "yes" ]; then
if [ $# -gt 0 ]; then
echo "error: -i don't take any parameters"
exit 1
fi
index
exit
fi

if [ $# -ne 1 ]; then
echo $USAGE
exit 1
fi

if [ ! -d .omega ]; then
echo "error: can't find .omega, run with -i to index first"
exit 1
fi
search $1

Blog Archive