#!/usr/bin/perl
#
#  makeRadarII.pl 
#
#  Program to make Radar level II products from the pieces. The pieces will 
#  be stored individually until the last piece arrives which will trigger a
#  collection/sort/concatenation into a final product.
#
# defaults
#	$datadir = "/data/ldm/pub/native/radar/level2";
#	$log = "/data/ldm/logs/makeRadarII.log";
# 
=Doc
Note from Tom Yoksas
It seems to me that the pieces of a Level II volume scan can be written
to disk and then assembled after all have been received.  The tricky
part is that one needs to receive the final piece to know how many
pieces there are/should be and then know how long to wait before
deciding that all of the pieces have not been received before exiting.

Here is a snippit of a 'notifyme' listing for KFTG today (it just so
happens that the sequence of pieces for this volume scan are in
order):

Oct 21 19:08:26 notifyme[3769] INFO:    12632 20091021190220.884 NEXRAD2 88001
L2-BZIP2/KFTG/20091021190220/88/1/S/V03/0
Oct 21 19:08:27 notifyme[3769] INFO:    41835 20091021190222.965 NEXRAD2 88002
L2-BZIP2/KFTG/20091021190220/88/2/I/V03/0
Oct 21 19:08:27 notifyme[3769] INFO:    60408 20091021190225.064 NEXRAD2 88003
L2-BZIP2/KFTG/20091021190220/88/3/I/V03/0
Oct 21 19:08:27 notifyme[3769] INFO:    61916 20091021190229.154 NEXRAD2 88004
L2-BZIP2/KFTG/20091021190220/88/4/I/V03/0
 ...
Oct 21 19:08:28 notifyme[3769] INFO:    18450 20091021190615.685 NEXRAD2 88067
L2-BZIP2/KFTG/20091021190220/88/67/I/V03/0
Oct 21 19:08:28 notifyme[3769] INFO:    16791 20091021190622.752 NEXRAD2 88068
L2-BZIP2/KFTG/20091021190220/88/68/I/V03/0
Oct 21 19:08:28 notifyme[3769] INFO:    16010 20091021190626.810 NEXRAD2 88069
L2-BZIP2/KFTG/20091021190220/88/69/I/V03/0
Oct 21 19:08:28 notifyme[3769] INFO:    16232 20091021190636.732 NEXRAD2 88070
L2-BZIP2/KFTG/20091021190220/88/70/E/V03/0

Notice that the last piece contains both the end of product indicator
and the exptected number of pieces:

.../KFTG/20091021190220/88/70/E/V03/0
                           ^  ^______ end piece indicator
                           |_________ piece number/exptected number of pieces

The product ID for the last piece shows that we should expect 70 pieces
for this particular product.  If the pieces were being written to
individual files, then it is easy enough to sort the files and cat them
into a single output file.


I suggest that the assembly process/"decoder" be kicked off upon
receipt of the end of product piece.  It would then sort the pieces it
has received and create an output file and delete individual pieces_if_
all of the pieces are accounted for.  If one or more piece is missing,
it could reschedule itself in 'n' minutes (1 minute?) to check again.
If a rescheduling approach is taken, then the assembler/"decoder"
should pass itself a count of how many times it has been rescheduled,
and give up after some reasonable number (like 5?) of attempts.  On the
final pass, it should create an output file that is as complete as it
is likely to be, and provide some indication that the file is not
complete (log file output, or a special suffix, etc.).

The above scheme is not foolproof: what happens if the missing piece(s)
are received after the assembly process has run the maximun number of
times?  I guess that the pieces could be resorted and then concatenated
into an output file that is then complete, but this would mean that one
would need to keep the pieces on disk for some undetermined amount of
time and kick off checking upon receipt of each piece.  It is my belief
that checking upon receipt of each piece would not work as the system
would get bogged down.  I say this because I tried logging the arrival
of each Nexrad Level III product, and the system rapidly became unusable
because of I/O load.

=cut

# process command line switches
while ($_ = $ARGV[0], /^-/) {
	 shift;
       last if /^--$/;
	     /^(-d)/ && ( $datadir = shift ) ;
	     /^(-l)/ && ( $log = shift ) ;
	     /^(-v)/ && $verbose++;
}
# process input parameters
if( $#ARGV == 3 ) {
	$stn = $ARGV[ 0 ] ;
	$yyyymmdd = $ARGV[ 1 ] ;
	$hhmm = $ARGV[ 2 ] ;
	$number = $ARGV[ 3 ] ;
} else {
	print "Not the correct number of parameters, need station yyyymmdd hhmm number_of_pieces\n";
	exit ( 1 );
}

if( ! $datadir ) {
	$datadir = "/data/ldm/pub/native/radar/level2";
}
if( ! $log ) {
	$log = "/data/ldm/logs/makeRadarII.log";
}
# go to dir and see if all pieces are there
chdir( "$datadir/$stn/$yyyymmdd/tmp" );

# Level2_KFTG_20091022_1240.ar2v
# L2-BZIP2/KFTG/20091021190220/88/70/E/V03/0

( @FILES ) = glob( "*$hhmm*" );
if( $#FILES == ($number -1) ) {
        # create product all pieces accounted for
	create();
	
	# check for older incomplete products
	if( -e "notReady" && ! -z "notReady" ) {
		open( IN, "notReady" );
		while( <IN> ) {
			( $stn, $yyyymmdd, $hhmm, $number, $count ) = split;
			( @FILES ) = glob( "*$hhmm*" );
			# all pieces accounted for so make product
			if( $#FILES == ($number -1) ) {
				create();
			# Not all pieces accounted for but still make product, only for an hour
			} elsif( $#FILES > -1 && $count < 15 ) {
				createpart();
				$count++;
				push( @PARTIAL, "$stn $yyyymmdd $hhmm $number $count\n" );
			#tried to make complete product for an hour, time to give up
			} elsif( $#FILES > -1 ) { 
				`/bin/echo "$stn $yyyymmdd $hhmm Pieces deleted after 15 attempts to create product" >> $log`;
        			`/bin/rm *$hhmm*`;
			}
		}
		close IN;
		open( OUT, ">notReady" );
		for( $i = 0; $i <= $#PARTIAL; $i++ ) {
			print OUT $PARTIAL[ $i ];
		}
		close OUT;
	} else {
		# check partial product with no ending piece
		( @FILES ) = glob( "$stn*" );
		if( $#FILES != -1 ) { # there's a partial file
			$FILES[ 0 ] =~ s#^(\w{4})_(\d{8})_(\d{4})##;
			$stn = $1;
			$yyyymmdd = $2;
			$hhmm = $3;
			$number = 100;
			createpart();
		}
	}
} else {
	`/bin/echo "$stn $yyyymmdd $hhmm $number 0" >> notReady`;
}
		
exit( 0 ) ; 

# sub create product
sub createpart
{
        # create product
        $file = "Level2_" . $stn . "_" . $yyyymmdd . "_" . $hhmm . ".ar2v";
	`/bin/echo "$stn $yyyymmdd $hhmm Partial product created" >> $log`;
	open( OUT, ">$file" );
	binmode( OUT );

        $part =  $stn . "_" . $yyyymmdd . "_" . $hhmm;
        for( $i = 1; $i <= 100; $i++ ) {
                $piece = $part . "_" . $i;
		next if( ! -e "$piece" );
		open( PART, "$piece" );
		binmode( PART );
		while(  read ( PART, $data, 8192) ) {
			print OUT $data;
		}
		close PART;
        }
        `mv $file ../$file`;
}


# sub create product
sub create
{
        # create product
        $file = "Level2_" . $stn . "_" . $yyyymmdd . "_" . $hhmm . ".ar2v";
	open( OUT, ">$file" );
	binmode( OUT );

        $part =  $stn . "_" . $yyyymmdd . "_" . $hhmm;
        for( $i = 1; $i <= $number; $i++ ) {
                $piece = $part . "_" . $i;
		open( PART, "$piece" );
		binmode( PART );
		while(  read ( PART, $data, 8192) ) {
			print OUT $data;
		}
		close PART;
        }
        `mv $file ../$file`;
        `/bin/rm $part*`;
}