#!/usr/bin/perl

##############################################################
# Paolo Ferragina @ August 2008
#
# Reads the pages from STDIN, determines the URL, sort them,
# and writes in two files the permutation function of the docIDs
# as well as their sizes.
# Files are: _PermutedPos_.txt e _PageSizes_.txt
#
# With "permutation function", I mean that in position j
# we find where docID j should go after the permutation.
##############################################################

use bytes;

$psize = 0;
$page_num = 0;
$row="";           ## currently parsed row

@url_array = ();
@psize_array = ();

open(OUT, ">temp.url");
open(OUT1, ">dummy");
while ($row = <STDIN>) {

    if ($row =~ m/^warc\/0\.9/) { 

	# Store the size of the previous page
	if ($page_num > 0){
	    $psize_array[$page_num-1] = $psize;
	    $psize = 0;
	}

	if ($page_num % 100000 == 0){
	    print "$page_num\n";
	}

	# Determine the URL
	@items = split(/ /,$row);
	$url = $items[3];
	($a,$b,$host,$suff_corrente) = split(/\//,$url,4);

	# Create the new URL
	$host1 = reverse $host;
	$url1 = $host1 . "/" . $suff_corrente;
	print OUT "$url1 $page_num\n";
	print OUT1 "$page_num\n";

	$page_num++;
    }

    chomp($row);
    $row .= "\n";
    $psize += length($row);
}

close(OUT);

# Last page
$psize_array[$page_num-1] = $psize;


# Sort according to URL, and then to PageID
print "sorting....\n";
`sort temp.url > temp.url1`;

# Build the permuted order
print "permuting...\n";
@permutation = ();
open(IN, "<temp.url1");
for($i=0; $i < $page_num; $i++){
    $row = <IN>;
    chomp($row);
    ($urlx,$pnum) = split(/ /,$row,2);
    $pnum += 0;
    $permutation[$pnum] = $i;
}
close(IN);


# Writes data for future processing
# Format:
#
# row 5: final position of page 5 + its size in bytes
print "writing...\n";
open(OUTPERM,">_PermutedPos_.txt");
open(OUTSIZE,">_PageSizes_.txt");
for($i=0; $i < $page_num; $i++){
    print OUTPERM "$permutation[$i]\n";
    print OUTSIZE "$psize_array[$i]\n";
}
close(OUTPERM);
close(OUTSIZE);
