#!/bin/bash

# tokenizer is available for use under the GNU General Public License.
# See http://www.cis.uni-muenchen.de/~wastl/misc/

# set the location of the tokenizer root here
TOKENIZER_HOME=~/Tools/tokenizer-1.0

$TOKENIZER_HOME/tokenizer -L en-u8 -SP -E ''