InstallationInstructions: makeindex.pl

$revisions=<<END;
Revisions

v0.17 - 08 Mar 07
 - Support of categories

v0.16 - 29 Feb 07
 - Using localized image namespace (e.g. Bild: for de)

v0.15 - 25 Feb 07
 - Changed deaccenting method to make it locale-independant
 - FIX locale-independant sort command

v0.10 - 20 Feb 07
 - Packing of image and math directories in at most 256 files for more convenient copy to device

v0.9 - 17 Feb 07
 - Added support for math notations (requires LaTeX)
 - Added some sanity checks on external dependencies
 - Added ability to take the archive as argument instead of the language. directories are created and archive is moved over as necessary
 - Better error management
 - Experimental support for automatic dump download (requires curl)
 - Better command line syntax to activate dump download, image download and math support

END


###########################
#
#  CONFIGURATION
#
###########################


# External Commands
#
$sortcmd="sort";       # IMPORTANT!!!! do NOT use the default windows sort command, it's way too slow. Use the Cygwin version
$curlcmd="curl";       #required for direct dump download
$latexcmd="latex";     #required for math support
$dvipscmd="dvipng";    #required for math support

#sortmethod: 
#1 for external sort command (preferred if you have the GNU sort),
#2 for an in-place mergesort !!very slow!! don't use...
#3 for an insertion sort
$sortmethod=1;


# Images
#
#imgsupport:
#0 for no image,
#1 for download and resize image,
#2 for image links (not supported yet)
# overriden by "image" or "img" command line option
$imgsupport = 0;

#Configuration for image handling
$thumbmediaprefix="http://upload.wikimedia.org/wikipedia/%s/thumb";
$fullmediaprefix="http://upload.wikimedia.org/wikipedia/%s";
$imgsize=120;

#extensions to download when $imgsupport is enabled
%allowedexts = (
        "jpg" => 1,
        "jpeg" => 1,
        "png" => 1,
        "svg" => 1, #svg is converted to png at download time
        "gif" => 1,
);


# Math
#
# requires a latex distrtibution (such as http://www.tug.org/protext/ for windows)
# overriden by "math" command line option
$mathsupport = 0;


# Download
#
$downloadsupport=0;
$downloadurl="http://download.wikimedia.org/LANGwiki/latest/LANGwiki-latest-pages-articles.xml.bz2";
$downloadname="LANGwiki-latest-pages-articles.xml.bz2";


# Categories
#
$catsupport=0;
$cathashlen=4;
$maxcatrefs=16;

# Namespaces
#
#List of namespaces to ignore, identified by their language-independent index. 0 = ignore, 1=include
%namespaces=(
      -2 => 0, #Media
      -1 => 0, #Special
      1 => 0, #Talk
      2 => 0, #User
      3 => 0, #User talk
      4 => 0, #Wikipedia
      5 => 0, #Wikipedia talk
      6 => 0, #Image
      7 => 0, #Image talk
      8 => 0, #MediaWiki
      9 => 0, #MediaWiki talk
      10 => 0, #>Template
      11 => 0, #>Template talk
      12 => 0, #>Help
      13 => 0, #>Help talk
      14 => 0, #>Category
      15 => 0, #>Category talk
      100 => 0, #Portail
      101 => 0, #Discussion Portail
      102 => 0, #Projet
      103 => 0, #Discussion Projet
      104 => 0, #R�f�rence
      105 => 0, #Discussion R�f�rence
);


# Pauses when launching external commands
# explorer.exe apparently leaks memory when you keep launching external commands...
$pausefreq=200;  #every nth launch of external cmd. Enter 0 for no pause
$pauselen=5;     #pause duration in seconds


# Packing
$packfile="pack.dat";
$idxfile="idx.dat";
$chunksize=10000000;

###########################
#
#  INIT
#
###########################

use Encode;
use File::Copy;
use utf8;

$usage = <<END;
$0: Offline wikipedia search index generator.
Usage:
$0 RootPath
 If dump file is already present in RootPath/bz2
 
$0 DumpFile
 Script will guess language and create appropriate directories

Options:
 --image/img: download image thumbnails from wikipedia
 --cat:       generate category indexes
 --math:      generate images for <math></math> equations (requires latex)
 --download:  download dump from wikimedia first (requires curl)
 N:           skip first N archive files (the resulting index will be incomplete but it's useful when restarting downloads / math processings)
 --help:      this message
 --version:   revision information
END

$rootpath=shift @ARGV or die "$usage\n";

while($arg=shift @ARGV) {
        if($arg =~ /--image|img/i) {$imgsupport=1;}
        if($arg =~ /--math/i) {$mathsupport=1;}
        if($arg =~ /^\d+$/) {$skipfirst=$arg;}
        if($arg =~ /--download/i) {$downloadsupport=1;}
        if($arg =~ /--help/i) {die "$usage\n";}
        if($arg =~ /--version/i) {die "$revisions\n";}
        if($arg =~ /--cat(egor(y|ies))?/i) {$catsupport=1;}
}

if($rootpath =~ m/^([\w\-]+)wiki\-.*\.xml\.bz2$/i) {
        $masterfile=$rootpath;
        $rootpath=$1;
}

$archivepath="$rootpath/bz2";
$indexpath="$rootpath/db";
$imgpath="$rootpath/img";
$mathpath="$rootpath/math";

MakeDirs($rootpath);
MakeDirs($archivepath);
MakeDirs($indexpath);

if(-e $masterfile) {
        move($masterfile, $archivepath);
}

if($imgsupport>0) {
        mkdir "$imgpath";
        use Digest::MD5 qw(md5_hex);
        use HTTP::Request;
        use LWP::UserAgent;
        $ua = new LWP::UserAgent;
        $ua->agent("iPhonePedia");
}

if($mathsupport>0) {
        mkdir "$mathpath";
        use Digest::MD5 qw(md5_hex);
}

$starttime=time;
@Titles=();
$pattern="*.xml.bz2";

$indexfile="Index.dat";
$bzip2recovercmd="bzip2recover";
$bzcatcmd="bzcat";

$imagecount=0;
$failedimagecount=0;
$foundimagecount=0;
$equationcount=0;
$foundequationcount=0;




###########################
#
#  SANITY
#
###########################

if(not `$sortcmd --help 2>&1` =~ /usage/i) {print "Could not find correct external sort.\nFalling back on internal algorithm (slower).\n"; $sortmethod=3;}
if($mathsupport and not (`$dvipscmd 2>&1` =~ m/usage/i && `$latexcmd --help 2>&1`=~ m/usage/i)) {die "Math support requires latex and dvipng.\nPlease install a LaTeX distribution to get those.\n";}
if(not `$bzcatcmd --help 2>&1` =~ /usage/i) {die "Cannot find bzcat (tried with $bzcatcmd).\n";}
if($downloadsupport and not `$curlcmd --help 2>&1` =~ /usage/i) {die "Curl is required for download support.\nPlease install it from http://curl.haxx.se\n";}

print "\n";
print "== Root path:     $rootpath\n";
print "== Archives path: $archivepath\n";
print "== Index path:    $indexpath\n";
print "== Image path:    $imgpath\n"  if($imgsupport);
print "== Math path:     $mathpath\n" if($mathsupport);
print "\n";


###########################
#
#  SPLIT ARCHIVE IF NEEDED
#
###########################

@list = glob("$archivepath/$pattern");

if(@list == 0) {
        $downloadurl =~ s/LANG/$rootpath/g;
        $downloadname =~ s/LANG/$rootpath/g;

        if($downloadsupport) {
                print "No archive found. Attempting to download it from wikimedia.\n";
                `curl $downloadurl -o \"$archivepath/$downloadname\"`;
                if(! -e "$archivepath/$downloadname") {die "Could not download archive.\n$downloadurl does not seem to exist. Please download manually from http://download.wikimedia.org\n";}
                @list = glob("$archivepath/$pattern");
        }
        else {
                die "Could not find any archive. Please make sure either the wikipedia dump or the splitted files are in $archivepath\nYou may want to download $downloadurl\n";
        }
}

$downloadtime=int((time - $starttime)/60);

if(@list == 1 && not $list[0] =~ /rec\d+/i) {
        $bigfile=$list[0];
        print "Only one archive found ($bigfile). Splitting.\n";

        print `$bzip2recovercmd $bigfile`;
        @list = glob("$archivepath/$pattern");
        if(@list > 1) {
                print "Success!\n";
                move($bigfile,".");
        }
}

$archivetime=int((time - $starttime)/60);

$allfiles=@list;



###########################
#
#  SCAN ARCHIVES
#
###########################

open OUTFILE,">$indexfile";
foreach $bzfile (@list) {
        next if ($bzfile eq $bigfile);
                
        if($bzfile =~ m/rec(\d+)(\D+)wiki/) {
                $idx=int($1);
                next if($idx<$skipfirst && $idx!=1);
                
                print "Scanning file number $idx/$allfiles\n";
                $content= `$bzcatcmd $bzfile`;
                #print "$content";

                if($idx==1) {
                        #extract namespace information
                        while($content =~ m|<namespace key="([\d\-]*)">(.*?)</namespace>|ig) {
                                unless($namespaces{$1}==1) {
                                        $ignorenamespaces{lc $2}=1;
                                        print "Ignoring namespace $2\n";
                                }
                                if($1==6) {
                                        #save localized image namespace name
                                        $imgnamespace=$2;
                                }
                                if($1==14) {
                                        #save category namespace name
                                        $catnamespace=$2;
                                }
                        }
                        if($imgnamespace) {
                                print "Using '$imgnamespace:' namespace for images\n";
                        }
                        if($catnamespace) {
                                print "Using '$catnamespace:' namespace for categories\n";
                        }
                }

                while($content =~ m|<title>([^<]+)</title>|g) {
                        $title=$1;
                        $offset=pos($content)-length("<title>$title</title>");
                        $skip=0;
                        foreach $ns (keys %ignorenamespaces) {if($title =~ /^$ns\:/i) {$skip=1;}}
                        next if($skip);
                        
                        #$title=decode("utf8", $title);
                        #$title=PlainAscii($title);
                                
                        #print "$title#$idx\n";
                                
                        if($sortmethod==1) {
                                print OUTFILE "$title#$idx-$offset\n";
                        }
                        elsif($sortmethod==2) {
                                push(@Titles, "$title#$idx-$offset");
                        }
                        elsif($sortmethod==3) {
                                print OUTFILE "$title#$idx-$offset\n";
                                #InsertSorted("$title#$idx-$offset");
                        }
                        
                }


                if($imgsupport>0) {
                        while($content =~ m/(\[|^|\=)\s*(image|$imgnamespace)\:(.*?)(\]|$)/mgi) {
                                $foundimagecount++;
                                $descr=$3;
                                if($descr=~ m/(\d+)px/) {$width=$1;}
                                else {$width=$imgsize;}
                                $descr =~ s/\|.*//;
                                $descr =~ s/\s+$//;
                                $descr =~ s/^\s+//;
                                
                                #skip past attempts for inexistent images
                                next if($failedimg{$descr}==1);
                                
                                print " (".GetProgress().")-Getting image #$imagecount: ";
                                $returned=ProcessImage($descr,$width);

                                if($returned==1) {
                                        print "ok\n";
                                        $imagecount++;
                                }
                                elsif($returned==2) {
                                        print "already downloaded\n";
                                }
                                else {
                                        print "failed\n";
                                        $failedimg{$descr}=1;
                                        $failedimagecount++;
                                }
                        }
                }
                
                if($mathsupport>0) {
                        while($content =~ m|\&lt;math\&gt;\s*(.*?)\s*\&lt;/math\&gt;|mgi) {
                                print " (".GetProgress().")-Rendering equation #$equationcount: ";
                                ProcessEquation($1);
                        }
                }

                if($catsupport>0) {
                        while($content =~ m|\[\[$catnamespace\:\s*([^\]]*?)\s*\]\]|mgi) {
                                $cat=$1;
                                $cat =~ s/[\n\|].*//gi;
                                $start=rindex($content, "<title>", pos($content));
                                $end=index($content, "</title>", $start);
                                if($start>0 && $end>0) {
                                        $title=substr($content, $start+7, $end-$start-7);
                                        #print " Adding '$title' to category '$cat'\n";
                                        push @{$catmembers{$cat}}, $title;
                                }
                        }
                }
        }
}


$exploretime=int((time - $starttime)/60);

if($imgsupport) {
        print "== Packing image dir:\n";
        PackImgDirectory();

        print "== Image stats:\n";
        print "  Detected  : $foundimagecount\n";
        print "  Downloaded: $imagecount\n";
        print "  Failed    : $failedimagecount\n";
}

if($mathsupport) {
        print "== Packing math dir:\n";
        PackMathDirectory();
        
        print "== Math stats:\n";
        print "  Found     : $foundequationcount\n";
        print "  Generated : $equationcount\n";
}

if($catsupport) {
        print "== Dumping categories:\n";
        Dumpcat();
        
        print "== Categories:\n";
        print "  Found     : ".(keys %catmembers)."\n";
}



###########################
#
#  SORT Index.dat
#
###########################

print "== Sorting\n";

$locale=$ENV{"LC_ALL"};
$ENV{"LC_ALL"}="C";

if($sortmethod==1) {
        close OUTFILE;
        print `$sortcmd -f -o \"$indexfile\" \"$indexfile\"`;
        #other options -t \'#\' -k 1,1 
}
elsif($sortmethod==2) {
        merge_sort(\@Titles, 0, scalar(@Titles)-1);
        foreach $elt (@Titles) {
                print OUTFILE "$elt\n";
        }
        close OUTFILE;
}
elsif($sortmethod==3) {
        close OUTFILE;
        open OUTFILE,"<$indexfile";
        while(<OUTFILE>) {
                chomp;
                InsertSorted("$_");
        }
        close OUTFILE;
        open OUTFILE,">$indexfile";
        foreach $elt (@Titles) {
                print OUTFILE "$elt\n";
        }
        close OUTFILE;
}

$ENV{"LC_ALL"}=$locale;


$sorttime=int((time - $starttime)/60);



###########################
#
#  BUILD FULL-TEXT INDEX
#
###########################

# Path to index database files
$HASH      = "${indexpath}/0_hash";
$HASHWORDS = "${indexpath}/0_hashwords";
$FINFO     = "${indexpath}/0_finfo";
$SITEWORDS = "${indexpath}/0_sitewords";
$WORD_IND  = "${indexpath}/0_word_ind";



#minimum word length to index
$min_length = 3;

# Index or not numbers (set   $numbers = ""   if you don't want to index numbers)
# You may add here other non-letter characters, which you want to index
$numbers = '0-9';

# Indexing scheme
# Whole word - 1
# Beginning of the word - 2
# Every substring - 3
$INDEXING_SCHEME = 1;

# List of stopwords
$stop_words = "and any are but can had has have her here him his how its not our out per she some than that the their them then there these they was were what you";


$HASHSIZE = 300001;


@stop_words=split(/\s+/,$stop_words);
foreach $stopword (@stop_words) {$stop_words_array{$stopword}=1; }


print "== Start indexing\n";


#DEFINE CONSTANTS
$cfn = 0;
$cwn = 0;

if(! -d "db") {
        mkdir("db",0755) or die("Can't create directory DB!!!");
        print "== Directory 'db' has been created\n";
}



mkdir($indexpath);
open(fp_FINFO,">$FINFO") or die("Can't open index file!\n");
open(fp_SITEWORDS ,">$SITEWORDS") or die("Can't open index file!\n");
open(fp_WORD_IND,">$WORD_IND") or die("Can't open index file!\n");

binmode fp_FINFO;
binmode fp_SITEWORDS;
binmode fp_WORD_IND;

print fp_FINFO "\x0A";

scan_list("$indexfile");

if ($cfn == 0) {
    die "No files are indexed\n";
}

print "== Computing word hash\n";
    $pos_sitewords = tell(fp_SITEWORDS);
    $pos_word_ind  = tell(fp_WORD_IND);
    $to_print_sitewords = "";
    $to_print_word_ind  = "";
    foreach $word (keys %words) {
        $value=$words{$word};
        $cwn++;
        $words_word_dum = pack("NN",$pos_sitewords+length($to_print_sitewords),
                                $pos_word_ind+length($to_print_word_ind));
        $to_print_sitewords .= "$word\x0A";
        $to_print_word_ind .= pack("N",length($value)/4).$value;
        $words{$word} = $words_word_dum;
        
        
        if (length($to_print_word_ind) > 32000) {
            print fp_SITEWORDS $to_print_sitewords;
            print fp_WORD_IND  $to_print_word_ind;
            $to_print_sitewords = "";
            $to_print_word_ind  = "";
            $pos_sitewords = tell(fp_SITEWORDS);
            $pos_word_ind  = tell(fp_WORD_IND);
        }

    }
    print fp_SITEWORDS $to_print_sitewords;
    print fp_WORD_IND  $to_print_word_ind;

close(fp_SITEWORDS);
close(fp_WORD_IND);

$indextime=int((time - $starttime)/60);

print "== Dumping hash\n";

build_hash();

print "== $cfn entries indexed\n";


$dumptime=int((time  - $starttime)/60);


###########################
#
#  CLEANUP & FINAL STATS
#
###########################
unlink <temp.*>;
unlink $indexfile;

print "\n";
print "- Download time: ${downloadtime}m\n" if($downloadsupport);
print "- Archive time:  ${archivetime}m\n";
print "- Explore time:  ${exploretime}m\n";
print "- Sort time:     ${sorttime}m\n";
print "- Index time:    ${indextime}m\n";
print "- Dump time:     ${dumptime}m\n";













#=====================================================================
#
#    Function risearch_hash($key)
#    Last modified: 16.04.2004 17:54
#
#=====================================================================

sub risearch_hash {
        my ($key)=@_;
    @chars = split(//,$key);
    for($i=0;$i<@chars;$i++) {
        $chars2[$i] = ord($chars[$i]);
    }

    $h = hex("00000000");
    $f = hex("0F000000");

    for($i=0;$i<@chars;$i++) {
                $h = ($h << 4) + $chars2[$i];
        if ($g = $h & $f) { $h ^= $g >> 24; };
        $h &= ~$g;
    }

    return $h;

}

#=====================================================================
#
#    Function index_file($html_text,$url)
#    Last modified: 15.07.2004 11:35
#
#=====================================================================

sub index_title {
        my ($textindex,$url) = @_;
        my %words_temp;


        $cfn++;

        #decode UTF8
        $textindex = decode_utf8($textindex);

        $textindex = RemoveHTMLentities($textindex);
        #$textindex =~ s/[^a-zA-Z�-��-�$numbers -]/ /g;
        $textindex =~ s/[^\w\d -]/ /g;
        $textindex =~ s/\s+/ /g;
        #$textindex = lc($textindex);
        $textindex = PlainAscii($textindex);

        #back to binary
        $textindex = encode_utf8($textindex);
        @words_temp=split(/\s+/,$textindex);

        $pos = tell(fp_FINFO);
        $pos = pack("N",$pos);
        print fp_FINFO "$url\x0A";
    

        foreach $word (@words_temp) {
                next if (length($word) < $min_length);
                next if ($stop_words_array{$word});
                $words{$word} .= $pos;
        
                #print "$word => ".$words{$word}."\n";
        }
}

#=====================================================================
#
#    Function build_hash()
#    Last modified: 16.04.2004 17:54
#
#=====================================================================

sub build_hash {

    for ($i=0; $i<$HASHSIZE; $i++) {$hash_array[$i] = "";};

    foreach $word (keys %words) {
        $value=$words{$word};
        if ($INDEXING_SCHEME == 3) { $subbound = length($word)-3; }
        else { $subbound = 1; }
        if (length($word)==3) {$subbound = 1;}
        $substring_length = 4;
        if ($INDEXING_SCHEME == 1) { $substring_length = length($word); }

        for ($i=0; $i<$subbound; $i++){
            $hash_value = abs(risearch_hash(substr($word,$i,$substring_length)) % $HASHSIZE);
            $hash_array[$hash_value] .= $value;
         }

    }

    open(fp_HASH, ">$HASH") or die("Can't open index file!");
    open(fp_HASHWORDS,">$HASHWORDS") or die("Can't open index file!");

        binmode fp_HASH;
        binmode fp_HASHWORDS;
        
    $zzz = pack("N", 0);
    print fp_HASHWORDS $zzz;
    $pos_hashwords = tell(fp_HASHWORDS);
    $to_print_hash = "";
    $to_print_hashwords = "";

    for ($i=0; $i<$HASHSIZE; $i++){
        $elt=$hash_array[$i];
        if ($elt eq "") {$to_print_hash .= $zzz;}
        else {
            $to_print_hash .= pack("N",$pos_hashwords + length($to_print_hashwords));
            $to_print_hashwords .= pack("N", length($elt)/8).$elt;
            
        }
        if (length($to_print_hashwords) > 64000) {
            print fp_HASH $to_print_hash;
            print fp_HASHWORDS $to_print_hashwords;
            $to_print_hash = "";
            $to_print_hashwords = "";
            $pos_hashwords  = tell(fp_HASHWORDS);
        }
    }
    print fp_HASH $to_print_hash;
    print fp_HASHWORDS $to_print_hashwords;


close(fp_HASH);
close(fp_HASHWORDS);


}
#=====================================================================




#=====================================================================
#
#    Function scan_files ($dir)
#    Last modified: 05.04.2005 16:41
#
#=====================================================================

sub scan_list {
        my ($dbfile)=@_;

        print "== Scanning $dbfile\n";
        open(FILE,$dbfile) or print "Cannot open $dbfile\n";

        while(<FILE>) {
                $line=$_;
                chomp $line;
                if($line =~ /^(.*)#.*/) {
                        #print "indexing entry: $line";
                        index_title($1,$line);
                }
        }
        close(FILE);
}

sub RemoveHTMLentities {
    my ($text) = @_;
    my (%entities, $key, $subst);
        
    %entities =  (  "&amp;"     =>  "&",
                    "&ndash;"   =>  "-",
                    "&lt;"      =>  "<",
                    "&gt;"      =>  ">",
                    "&quote;"   =>  "\"",
                    "&quot;"    =>  "\'"
                 );
    
    foreach $key (keys %entities)
        {
        $subst = $entities{$key};
        $text =~ s/$key/$subst/g;
        }
        
    return $text;
}

sub GetMediaHash {
    my ($medianame) = @_;
    my ($md5, $path, $fileprefix);

    # First-capitalize Unpack %xx hex-encoded characters, and convert resulting spaces in "_"
    $medianame = ucfirst $medianame;
    $medianame =~ s/%(..)/pack("c",hex($1))/ge;
    $medianame =~ s/\s/_/g;
    
    # Remove <> signs, on some systems we cannot write files with them
    $medianame =~ s/\>/_/g;
    $medianame =~ s/\</_/g;
    # replace remaining unicase (if there are errors in the media names)
    #$medianame =~ s/[^\p{Latin}\p{NP}]/_/g;

    #print "$medianame -";  
    #use Encode 'from_to';
    #from_to($medianame,"utf-8","iso-8859-15");
    #$medianame = pack("C*",unpack("U*", $medianame));
    $md5 = md5_hex($medianame);

    return $md5;
}

sub ProcessImage {
        my ($title,$width)=@_;
        my ($path, $onlinepath, $outpath, $outfile, $langpath, $ext);
        
        #reject unsupported extensions
        if($title =~ m/\.([^\.]+)$/i) {$ext=lc $1;}