##############################################################
# Paolo Ferragina @ August 2008
#
# This perl script compresses a large file into blocks
# according to a standard string compressor specified
# in input. The file "_block_.txt" is used to store
# the current block in order to compress it.
# 
# It assumes that the input file is in WARC format, and
# is fetched from STDIN. We discard the WARC-header during
# the compression process.
# 
# In file "stats.txt" we report some statistics of the process.
# The file "_block_.txt" is used as temporary file to store
# the block to be compressed.
# The file "_compr-offsets_.out" stores textually the size of
# the compressed blocks.
# The file "_compr-file_.out" stores the compressed blocks
# one after the other.
##############################################################

if (scalar(@ARGV) < 4) {
    print "\n----- Please keep attention, you need to specify 4 params:\n";
    print "  blockSize    (in Mbs)\n";
    print "  Compressor Type ([0-20] = gzip -9, bzip2 -9, PPMdj1, \n";    
    print "                           bzip+MultiTableHuff, bzip+Arith, bzip+rle+ac\n";
    print "                           LZopt, LZRightMost \n";    
    print "                           PPMonster \n";    
    print "                           vcdiff (emptyDict+ gzip -9),\n";
    print "                           vcdiff (emptyDict + booster) \n";    
    print "                           vcdiff (emptyDict + p7zip) \n";    
    print "                           vcdiff (emptyPage+lzopt),\n"; 
    print "                           vcdiff (emptyDict + gzip -7) \n";    
    print "                           bwtp   \n";    
    print "                           p7zip  \n";    
    print "                           BMI+GZ \n";    
    print "                           RZIP   \n";    
    print "                           BMI+p7zip\n";    
    print "                           BMI+Booster\n";    
    print "                           Vcodex+P7zip (200Mb window)\n";    
    print "  numBlocks to compress (0 = the whole file)\n";
    print "  write compressed or not (1 or 0)\n\n";
    print "the data to be compressed must be in WARC format and is fetched from STDIN.\n\n";
    exit;
    }
  
#########################################
# Load the input paramenters
# and initialize the other variables
###############################################
$sizeMaxBlock = shift @ARGV; # in Mbytes
$sizeMaxBlock = $sizeMaxBlock * 1000000; # in bytes

$comprType = shift @ARGV; # 0-4 for now

if (($comprType < 0) || ($comprType > 20)) {
    print "comprType must be in [0-18]!\n\n";
    exit;
}

$numTotBlocks = shift @ARGV; # 0 = total file
if ($numTotBlocks < 0) {
    print "numTotBlocks must be a non negative value!\n\n";
    exit;
}

$writeOut = shift @ARGV; # 0 or 1
if (($writeOut < 0) || ($writeOut > 1)){
    print "You must specify 1 (write) or 0 (not write) the compressed output!\n\n";
    exit;
}

#######################################################################
# Useful variables
$unComprSize = 0;  ## size of the file to be compressed
$comprSize = 0;    ## size of the final compressed file
$sizeUnCTmp = 0;      ## tmp var to store the value of the UNcompr block    
$sizeCTmp = 0;      ## tmp var to store the value of the compr block    
$numBlocks = 0;    ## number of (currently) parsed blocks
$numPages = 0;     ## number of (currently) parged pages
$sizeCurrBlock=0;  ## size of the currently parsed block
$row="";           ## currently parsed row

$min_blk_ratio = 101;  ## for statistics
$max_blk_ratio = 0;    ## for statistics


### Timings
$initTime = time(); # in secs from the epoch
$prevTime = time(); 
$currTime = 0;
$timeComprBlock = 0;
$timeTot = 0;

######################################################
# File containing the block to compress and the 
# compressed block sizes for future decompression
######################################################
`rm -f _compr-file_.out _compr-sizes_.out _block_.txt _aa_`;
open(BLOCKFILE,">_block_.txt");

open(SIZES,">_compr-sizes_.out");
print SIZES "$comprType\n"; #so that it must not be passed in decompr

## File for the stats
open(STATS,">stats.txt");

printf(STATS "\n\n******************************************************\n");
printf(STATS "******* NOTE:\n");
printf(STATS "******* Load table in Excel, changing decimal . in ,\n");
printf(STATS "******************************************************\n\n");


printf(STATS "Max block size = %.2f\n",$sizeMaxBlock/1000000.0);
printf(STATS "Compressor Type = %d\n", $comprType);
printf(STATS "   ([0-20] =  gzip -9, bzip2 -9, PPMdj1, \n");    
printf(STATS "             bzip+MultiTableHuff, bzip+Arith, bzip+rle+ac\n");    
printf(STATS "             LZopt, LZRightMost \n");    
printf(STATS "             PPMonster \n");    
printf(STATS "             vcdiff (emptyDict + gzip -9),\n");
printf(STATS "             vcdiff (emptyDict + booster) \n");    
printf(STATS "             vcdiff (emptyDict + p7zip) \n");    
printf(STATS "             vcdiff (emptyPage+lzopt),\n"); 
printf(STATS "             vcdiff (emptyDict + gzip -7) \n");    
printf(STATS "             bwtp   \n");    
printf(STATS "             p7zip  \n");    
printf(STATS "             BMI+GZ \n");    
printf(STATS "             RZIP \n");    
printf(STATS "             BMI+p7zip\n");    
printf(STATS "             BMI+Booster\n");    
printf(STATS "             Vcodex+p7zip\n");    
printf(STATS "Num blocks to process = %d (0 : all)\n\n",$numTotBlocks);
printf(STATS "---------------- Table ----------------\n");
printf(STATS "UnCompr, Compr, Tot ratio, Block Ratio\n");
printf(STATS "  (Mb)    (Mb)    (perc)      (perc)  \n");
printf(STATS "---------------------------------------\n\n");


print "\n\nProcess started.....\n\n\n";
`echo "" > _dict_.txt`;

printf("Creates block \# %d..... ",$numBlocks+1);
while ($row = <STDIN>) {

    #### A new page is met
    if ($row =~ m/^warc\/0\.9/) { 
	$numPages++;    # counts the number of pages

        ## rough scheme to delete the WARC header
	$count=2;  
	while ($count > 0) { ### it has 2 "empty" lines
	    $row = <STDIN>;
	    if ((length $row) < 3) {$count--;}  ### Case of "empty" line
	}
        ###### WARC-header has been discarded


	## If we reached the maximum allowed block size
	if ($sizeCurrBlock > $sizeMaxBlock){

	    # timings generation
	    $currTime = time(); # in secs from the epoch
	    printf(" done! (%d secs)\n", $currTime - $prevTime);
	    $prevTime = $currTime;


	    # We have a new block, and count it
	    $numBlocks++;


	    # Stat the block
	    close(BLOCKFILE);   
	    $sizeUnCTmp = -s "_block_.txt";
	    $unComprSize += $sizeUnCTmp;


	    # Perl does not support natively the SWITCH....
	    printf("Compresses it (via %d on %d Mbs)",$comprType,$sizeMaxBlock/1000000);
	    if($comprType == 0)
	    { `gzip -9 _block_.txt ; mv _block_.txt.gz _block_.out`; } 
	    elsif($comprType == 1) 
	    { `bzip2 -9 _block_.txt; mv _block_.txt.bz2 _block_.out`; }
	    elsif($comprType == 2)
	    { `./ppmdi e -r1 -f_block_.out _block_.txt`; }
	    elsif($comprType == 3)
	    { `./bst -fm -a6 _block_.txt; mv _block_.txt.bst _block_.out`; }
	    elsif($comprType == 4)
	    { `./bst -fm -a2 -y16384 -z64 _block_.txt; mv _block_.txt.bst _block_.out`; }
	    elsif($comprType == 5)
	    { `./bst -f -a7 -y65536 -z256 _block_.txt; mv _block_.txt.bst _block_.out`; }
	    elsif($comprType == 6)
	    { `./lzopt -r 0 _block_.txt; mv _block_.txt.opt _block_.out`; }
	    elsif($comprType == 7)
	    { `./lzopt -r 1 _block_.txt; mv _block_.txt.grm _block_.out`; }
	    elsif($comprType == 8)
	    { `./ppmdi e -r1 -o16 -m256 -f_block_.out _block_.txt`; }
	    elsif($comprType == 9)
	    { `vcdiff encode --target_matches --dictionary _dict_.txt < _block_.txt > _aa_; gzip -9 -c _aa_ >_block_.out`; }
	    elsif($comprType == 10)
	    { `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; ./bst -fm -a6 _aa_; mv _aa_.bst _block_.out`; }
	    elsif($comprType == 11)
	    { `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; ./7za a -mx=9 -md=30 _block_.out _aa_;`; }
	    elsif($comprType == 12)
	    { `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt >  _aa_; ./lzopt -r 0 _aa_; mv _aa_.opt _block_.out`; }
	    elsif($comprType == 13)
	    { `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; gzip -7 -c _aa_ > _block_.out`; }
	    elsif($comprType == 14)
	    { `./bwtp -t 1 -b 16000000 _block_.txt; mv _block_.txt.bwp _block_.out;`; }
	    elsif($comprType == 15) #cambiato in 128M
	    { `./7za a -mx=9 -md=128m _block_.out _block_.txt; `; }
	    elsif($comprType == 16)
	    { `./BMIcompress 50 <_block_.txt | gzip -9 -c > _block_.out;`; }
	    elsif($comprType == 17)
	    { `./rzip -9 -f _block_.txt -o _block_.out; `; }
	    elsif($comprType == 18)
	    { `./BMIcompress 50 <_block_.txt > _block_.tmp; ./7za a -mx=9 -md=128m _block_.out _block_.tmp;`; }
	    elsif($comprType == 19)
	    { `./BMIcompress 50 <_block_.txt > _aa_; ./bst -fm -a6 _aa_; mv _aa_.bst _block_.out;`; }
	    elsif($comprType == 20)
	    { `./vczip -w200m -plain -mdelta,huffgroup <_block_.txt > _block_.tmp; ./7za a -mx=9 -md=30 _block_.out _block_.tmp;`; }
	    else { print "The compressor type must be in [0-20]!"; exit; }


	    # Writes the offset and appends the compressed block
	    $sizeCTmp = -s "_block_.out"; 

	    if($sizeCTmp == 0) { print "ZERO !!!\n\n"; exit; }

	    if ($writeOut == 1) {
		print SIZES "$sizeCTmp\n";
		`cat _block_.out >> _compr-file_.out`;
	    }

	    # timings compression
	    $currTime = time(); # in secs from the epoch
	    printf(" done! (%d secs)\n", $currTime - $prevTime);
	    $timeComprBlock += ($currTime - $prevTime);
	    $prevTime = $currTime;

	    # Update the total compressed size
	    $comprSize += $sizeCTmp;

	    # removes the temporary files
	    `rm -f _block_*; rm -f _aa_`;

	    # Some stats
	    $x = $unComprSize/1000000.0;
	    $y = $comprSize/1000000.0;
	    $ratio = (100.0 * $comprSize) / $unComprSize;
	    $blockRatio = 100.0 * ($sizeCTmp/$sizeUnCTmp);

	    if ($blockRatio < $min_blk_ratio){ 
		$min_blk_ratio = $blockRatio;
	    } elsif ($blockRatio > $max_blk_ratio) {
		$max_blk_ratio = $blockRatio; }

            ## Print into the STAT file
	    printf(STATS "%.2f\t%.2f\t%.2f\t%.2f\n",$x,$y,$ratio,$blockRatio);

            # Temporary printing (at video)
	    print "Num parsed pages = $numPages\n";
	    printf("Compression ratio for this block = %.2f\%\n",$blockRatio);
	    printf("Total compressed ratio = %.2f\%\n",$ratio);
	    printf("Total uncompressed size = %.2f Mbs\n",$x);
	    printf("Total compressed size = %.2f Mbs (%.1f)\n\n",$y,$comprSize);



	    # Stop if we parsed the requested num of blocks
	    # Notice that starts from 1, so if numTotBlocks==0....
	    if ($numBlocks == $numTotBlocks) {

		print "\n   Processed all requested $numBlocks blocks!\n\n";
		printf(STATS "\n\n\n--------------- Overall statistics---------------\n");
		printf(STATS "Block size = %.2f Mbs\n", $sizeMaxBlock/1000000.0);
		printf(STATS "Num blocks processed = $numBlocks\n");
		printf(STATS "Num parsed pages = $numPages\n");
		printf(STATS "Total uncompressed size = %.2f Mbs (no WARC-header)\n",$x);
		printf(STATS "Total compressed size = %.2f Mbs (%.1f)\n",$y,$comprSize);
		printf(STATS "Total compressed ratio = %.2f\%\n",$ratio);
		$timeTot = time() - $initTime;
		if ($timeTot == 0) { $timeTot = 1; }
		printf(STATS "Total time for block compression = %d secs\n",$timeComprBlock);
		printf(STATS "Total time = %d secs\n",$timeTot);
		printf(STATS "Ratio compr/total time = %.2f \%\n\n", 100 * ($timeComprBlock/$timeTot));
		# Printing at video
		printf("Total time for block compression = %d secs\n",$timeComprBlock);
		printf("Total time = %d secs\n",$timeTot);
		printf("Ratio compr/total time = %.2f \%\n", 100 * ($timeComprBlock/$timeTot));

		exit;
	    }

	    # re-opens the temporary files for the next block
	    printf("Creates block \# %d..... ",$numBlocks+1);
	    $sizeCurrBlock = 0;
	    open(BLOCKFILE,">_block_.txt");
	}
    }

    # In all cases (also if we flushed a block)
    # We print out the row we have in cache
    print BLOCKFILE $row;
    $sizeCurrBlock += length $row;
}


#############################################################
## We need to process the last block (which is suspeded)

$numBlocks++;

# timings generation
$currTime = time(); # in secs from the epoch
printf(" done! (%d secs)\n", $currTime - $prevTime);
$prevTime = $currTime;

# Stat the block
close(BLOCKFILE);   
$sizeUnCTmp = -s "_block_.txt";
$unComprSize += $sizeUnCTmp;



# Perl does not support natively the SWITCH....
print "Compresses it..... ";


if($comprType == 0)
{ `gzip -9 _block_.txt ; mv _block_.txt.gz _block_.out`; } 
elsif($comprType == 1) 
{ `bzip2 -9 _block_.txt; mv _block_.txt.bz2 _block_.out`; }
elsif($comprType == 2)
{ `./ppmdi e -r1 -f_block_.out _block_.txt`; }
elsif($comprType == 3)
{ `./bst -fm -a6 _block_.txt; mv _block_.txt.bst _block_.out`; }
elsif($comprType == 4)
{ `./bst -fm -a2 -y16384 -z64 _block_.txt; mv _block_.txt.bst _block_.out`; }
elsif($comprType == 5)
{ `./bst -f -a7 -y65536 -z256 _block_.txt; mv _block_.txt.bst _block_.out`; }
elsif($comprType == 6)
{ `./lzopt -r 0 _block_.txt; mv _block_.txt.opt _block_.out`; }
elsif($comprType == 7)
{ `./lzopt -r 1 _block_.txt; mv _block_.txt.grm _block_.out`; }
elsif($comprType == 8)
{ `./ppmdi e -r1 -o16 -m256 -f_block_.out _block_.txt`; }
elsif($comprType == 9)
{ `vcdiff encode --target_matches --dictionary _dict_.txt < _block_.txt > _aa_; gzip -9 -c _aa_ >_block_.out`; }
elsif($comprType == 10)
{ `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; ./bst -fm -a6 _aa_; mv _aa_.bst _block_.out`; }
elsif($comprType == 11)
{ `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; ./7za a -mx=9 -md=30 _block_.out _aa_;`; }
elsif($comprType == 12)
{ `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt >  _aa_; ./lzopt -r 0 _aa_; mv _aa_.opt _block_.out`; }
elsif($comprType == 13)
{ `vcdiff encode --target_matches -dictionary _dict_.txt < _block_.txt > _aa_; gzip -7 -c _aa_ > _block_.out`; }
elsif($comprType == 14)
{ `./bwtp -t 1 -b 16000000 _block_.txt; mv _block_.txt.bwp _block_.out;`; }
elsif($comprType == 15)
{ `./7za a -mx=9 -md=128m _block_.out _block_.txt; `; }
elsif($comprType == 16)
{ `./BMIcompress 50 <_block_.txt | gzip -9 -c > _block_.out;`; }
elsif($comprType == 17)
{ ` ./rzip -9 -f _block_.txt -o _block_.out; `; }
elsif($comprType == 18)
{ `./BMIcompress 50 <_block_.txt > _block_.tmp; ./7za a -mx=9 -md=128m _block_.out _block_.tmp;`; }
elsif($comprType == 19)
{ `./BMIcompress 50 <_block_.txt > _aa_; ./bst -fm -a6 _aa_; mv _aa_.bst _block_.out;`; }
elsif($comprType == 20)
{ `./vczip -w200m -mdelta,huffgroup <_block_.txt > _block_.tmp; ./7za a -mx=9 -md=30 _block_.out _block_.tmp;`; }
else { print "The compressor type must be in [0-20]!"; exit; }


# Writes the offset and appends the compressed block
$sizeCTmp = -s "_block_.out"; 
if ($writeOut == 1) {
    print SIZES "$sizeCTmp\n";
    `cat _block_.out >> _compr-file_.out`;
}


# removes the temporary files
`rm -f _block_.txt; rm -f _block_.* ; rm -f _aa_`;


# timings compression
$currTime = time(); # in secs from the epoch
printf(" done! (%d secs)\n", $currTime - $prevTime);
$timeComprBlock += ($currTime - $prevTime);
$prevTime = $currTime;


# Update the total compressed size
$comprSize += $sizeCTmp;


close(SIZES);

# Some stats
$x = $unComprSize/1000000.0;
$y = $comprSize/1000000.0;
$ratio = (100.0 * $comprSize) / $unComprSize;
$blockRatio = 100.0 * ($sizeCTmp/$sizeUnCTmp);

if ($blockRatio < $min_blk_ratio){ $min_blk_ratio = $blockRatio;
} elsif ($blockRatio > $max_blk_ratio) { $max_blk_ratio = $blockRatio; }

## Print into the STATS file
printf(STATS "%.2f\t%.2f\t%.2f\t%.2f\n",$x,$y,$ratio,$blockRatio);

printf(STATS "\n\n\n--------------- Overall statistics---------------\n");
printf(STATS "Block size = %.2f Mbs\n", $sizeMaxBlock/1000000.0);
printf(STATS "Num blocks processed = $numBlocks\n");
printf(STATS "Num parsed pages = $numPages\n");
printf(STATS "Total uncompressed size = %.2f Mbs (no WARC-header)\n",$x);
printf(STATS "Total compressed size = %.2f Mbs\n",$y);
printf(STATS "Total compressed ratio = %.2f\%\n",$ratio);
printf(STATS "Compression thoroughput %.2f MB/secs\n",$x/$timeComprBlock);
printf(STATS "Minimum Block Compression Ratio = %.2f\%\n",$min_blk_ratio);
printf(STATS "Maximum Block Compression Ratio = %.2f\%\n",$max_blk_ratio);

$timeTot = time() - $initTime;
if ($timeTot == 0) { $timeTot = 1; }
printf(STATS "Total time for block compression = %d secs\n",$timeComprBlock);
printf(STATS "Total time = %d secs\n",$timeTot);
printf(STATS "Ratio compr/total time = %.2f \%\n", 100 * ($timeComprBlock/$timeTot));
close(STATS);

#######################################################################




##### Summary printing (at video) of the overall process
printf("Block size = %.2f Mbs\n", $sizeMaxBlock/1000000.0);
print "Num blocks processed = $numBlocks\n";
print "Num parsed pages = $numPages\n";
printf("Total uncompressed size = %.2f Mbs (no WARC-header)\n",$x);
printf("Total compressed size = %.2f Mbs\n",$y);
printf("Total compressed ratio = %.2f\%\n",$ratio);
printf("Compression thoroughput %.2f MB/secs\n",$x/$timeComprBlock);
printf("Minimum Block Compression Ratio = %.2f\%\n",$min_blk_ratio);
printf("Maximum Block Compression Ratio = %.2f\%\n",$max_blk_ratio);
printf("Total time for block compression = %d secs\n",$timeComprBlock);
printf("Total time = %d secs\n",$timeTot);
printf("Ratio compr/total time = %.2f \%\n", 100 * ($timeComprBlock/$timeTot));





