#!/usr/bin/perl

##############################################################
# Paolo Ferragina @ August 2008
#
# Reads the pages from STDIN, assigns a random string, sorts them,
# and writes in two files the permutation function of the docIDs
# as well as their sizes.
# Files are: _PermutedPos_.txt e _PageSizes_.txt
#
# With "permutation function", I mean that in position j
# we find where docID j should go after the permutation.
##############################################################

use bytes;

$psize = 0;
$page_num = 0;
$row="";           ## currently parsed row

@url_array = ();
@psize_array = ();

while ($row = <STDIN>) {

    chomp($row);
    $row .= "\n";

    if ($row =~ m/^warc\/0\.9/) { 

	# Store the size of the previous page
	if ($page_num > 0){
	    $psize_array[$page_num-1] = $psize;
	    $psize = 0;
	}

	if ($page_num % 100000 == 0){
	    print "$page_num\n";
	}

	$url1 = int(rand(1000000001)) . "  ";
	$url_array[$page_num] = "$url1 $page_num";

	$page_num++;
    }
    
    $psize += length($row);
}

# Last page
$psize_array[$page_num-1] = $psize;

# Sort according to URL, and then to PageID
@url_sorted = sort @url_array;

# Build the permuted order

@permutation = ();
for($i=0; $i < $page_num; $i++){
    ($urlx,$pnum) = split(/ /,$url_sorted[$i],2);
    $permutation[$pnum] = $i;
}

# Writes data for future processing
# Format:
#
# row 5: final position of page 5 + its size in bytes
open(OUTPERM,">_PermutedPos_.txt");
open(OUTSIZE,">_PageSizes_.txt");
for($i=0; $i < $page_num; $i++){
    print OUTPERM "$permutation[$i]\n";
    print OUTSIZE "$psize_array[$i]\n";
}
close(OUTPERM);
close(OUTSIZE);
