Generated: Mon Aug 29 19:34:15 2016 from cfcsvlog.pl 2016/08/20 16.9 KB. text copy
#!/usr/bin/perl -w # NAME: cfcsvlog.pl # AIM: Read and analyse a crossfeed log use strict; use warnings; use File::Basename; # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] ) use Date::Parse; use Cwd; my $os = $^O; my $perl_dir = '/home/geoff/bin'; my $PATH_SEP = '/'; my $temp_dir = '/tmp'; if ($os =~ /win/i) { $perl_dir = 'C:\GTools\perl'; $temp_dir = $perl_dir; $PATH_SEP = "\\"; } unshift(@INC, $perl_dir); require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl' Check paths in \@INC...\n"; require 'fg_wsg84.pl' or die "Unable to load fg_wsg84.pl ...\n"; # log file stuff our ($LF); my $pgmname = $0; if ($pgmname =~ /(\\|\/)/) { my @tmpsp = split(/(\\|\/)/,$pgmname); $pgmname = $tmpsp[-1]; } my $outfile = $temp_dir.$PATH_SEP."temp.$pgmname.txt"; open_log($outfile); # user variables my $VERS = "0.0.5 2015-01-09"; my $load_log = 0; my $in_file = ''; my $verbosity = 0; my $out_file = ''; my $out_xg = $temp_dir.$PATH_SEP."tempcsv.xg"; my $def_min_dist = 2; my $def_min_secs = 60; my $def_min_secs2 = 15 * 60; # 15 mins my $def_min_elap = 15; # ### DEBUG ### my $debug_on = 1; my $def_file = 'F:\FGx\crossfeed-dailies\csv\flights-2016-08-19.csv'; ##my $def_file = 'D:\FG\cflogs\cflogs\flights-2016-08-09.csv'; ##my $def_file = 'D:\FG\cflogs\cflogs\flights-2016-08-08.csv'; ##my $def_file = 'D:\FG\cflogs\cflogs\flights-2016-07-30.csv'; ### program variables my @warnings = (); my $cwd = cwd(); my $MPS2KT = 1.94384; # meters per second to knots my $SG_EPSILON = 0.000001; my $NM2KM = 1.852; # Nautical Miles to Kilometers sub VERB1() { return $verbosity >= 1; } sub VERB2() { return $verbosity >= 2; } sub VERB5() { return $verbosity >= 5; } sub VERB9() { return $verbosity >= 9; } sub show_warnings($) { my ($val) = @_; if (@warnings) { prt( "\nGot ".scalar @warnings." WARNINGS...\n" ); foreach my $itm (@warnings) { prt("$itm\n"); } prt("\n"); } else { prt( "\nNo warnings issued.\n\n" ) if (VERB9()); } } sub pgm_exit($$) { my ($val,$msg) = @_; if (length($msg)) { $msg .= "\n" if (!($msg =~ /\n$/)); prt($msg); } show_warnings($val); close_log($outfile,$load_log); exit($val); } sub prtw($) { my ($tx) = shift; $tx =~ s/\n$//; prt("$tx\n"); push(@warnings,$tx); } sub get_time_stg($) { my $elap = shift; my $negative = 0; my $units = ''; if ($elap < 0) { $negative = 1; $elap = -$elap; } if ( !($elap > 0.0) ) { return "0.0 s"; } if ($elap < 1e-21) { #// yocto - 10^-24 $elap *= 1e+21; $units = "ys"; } elsif ($elap < 1e-18) { #// zepto - 10^-21 $elap *= 1e+18; $units = "zs"; } elsif ($elap < 1e-15) { #// atto - 10^-18 $elap *= 1e+15; $units = "as"; } elsif ($elap < 1e-12) { #// femto - 10^-15 $elap *= 1e+12; $units = "fs"; } elsif ($elap < 1e-9) { #// pico - 10^-12 $elap *= 1e+9; $units = "ps"; } elsif ($elap < 1e-6) { #// nanosecond - one thousand millionth (10?9) of a second $elap *= 1e+6; $units = "ns"; } elsif ($elap < 1e-3) { #// microsecond - one millionth (10?6) of a second $elap *= 1e+3; $units = "us"; } elsif ($elap < 1.0) { #// millisecond $elap *= 1000.0; $units = "ms"; } elsif ($elap < 60.0) { $units = "s"; } else { my $secs = int($elap + 0.5); my $mins = int($secs / 60); $secs = ($secs % 60); if ($mins >= 60) { my $hrs = int($mins / 60); $mins = $mins % 60; if ($hrs >= 24) { my $days = int($hrs / 24); $hrs = $hrs % 24; return sprintf("%d days %2d:%02d:%02d hh:mm:ss", $days, $hrs, $mins, $secs); } else { return sprintf("%2d:%02d:%02d hh:mm:ss", $hrs, $mins, $secs); } } else { return sprintf("%2d:%02d mm:ss", $mins, $secs); } } my $res = ''; if ($negative) { $res = '-'; } $res .= "$elap $units"; return $res; } sub mycmp_decend_n { return -1 if ($a < $b); return 1 if ($a > $b); return 0; } sub get_ll_double($) { my $deg = shift; my $dbl = sprintf("%12.6f",$deg); return $dbl; } sub get_alt_stg($) { my $alt = shift; $alt = int($alt + 0.5); if ($alt > 10000) { $alt = 'FL'.int($alt / 100); #$alt = sprintf("%6d",$alt); } else { $alt = sprintf("%8d",$alt); } $alt = ' '.$alt while (length($alt) < 8); return $alt; } sub get_spd_stg($) { my $spd = shift; $spd = int($spd + 0.5); $spd = sprintf("%4d",$spd); return $spd; } sub get_hdg_stg($) { my $hdg = shift; $hdg = int($hdg + 0.5); $hdg = sprintf("%03d",$hdg); return $hdg; } sub get_decimal_stg($$$) { my ($dec,$il,$dl) = @_; my (@arr); if ($dec =~ /\./) { @arr = split(/\./,$dec); if (scalar @arr == 2) { $arr[0] = " ".$arr[0] while (length($arr[0]) < $il); $dec = $arr[0]; if ($dl > 0) { $dec .= "."; $arr[1] = substr($arr[1],0,$dl) if (length($arr[1]) > $dl); $dec .= $arr[1]; } } } else { $dec = " $dec" while (length($dec) < $il); if ($dl) { $dec .= "."; while ($dl--) { $dec .= "0"; } } } return $dec; } sub get_sg_dist_stg($) { my ($sg_dist) = @_; my $sg_km = $sg_dist / 1000; my $sg_im = int($sg_dist); my $sg_ikm = int($sg_km + 0.5); my $dlen = 5; # if (abs($sg_pdist) < $CP_EPSILON) my $sg_dist_stg = ""; if (abs($sg_km) > $SG_EPSILON) { # = 0.0000001; # EQUALS SG_EPSILON 20101121 if ($sg_ikm && ($sg_km >= 1)) { $sg_km = int(($sg_km * 10) + 0.05) / 10; #$sg_dist_stg .= get_decimal_stg($sg_km,5,1)." km"; $sg_dist_stg .= get_decimal_stg($sg_km,($dlen - 2),1)." km"; } else { #$sg_dist_stg .= "$sg_im m, <1km"; #$sg_dist_stg .= get_decimal_stg($sg_im,7,0)." m."; $sg_dist_stg .= get_decimal_stg($sg_im,$dlen,0)." m."; } } else { #$sg_dist_stg .= "0 m"; #$sg_dist_stg .= get_decimal_stg('0',7,0)." m."; $sg_dist_stg .= get_decimal_stg('0',$dlen,0)." m."; } return $sg_dist_stg; } sub process_in_file($) { my ($inf) = @_; if (! open INF, "<$inf") { pgm_exit(1,"ERROR: Unable to open file [$inf]\n"); } my @lines = <INF>; close INF; my $lncnt = scalar @lines; prt("Processing $lncnt lines, from [$inf]...\n"); my ($line,$inc,$lnn,@arr); my ($fid,$callsign,$lat,$lon,$alt_ft,$model,$spd_kts,$hdg,$dist_nm,$upd,$tsecs,$ra,$cnt,$ra2); my ($upd2,$be,$ee,$elap,$tm,$msg); my $curr_upd = ''; $lnn = 0; my %hash = (); my $block_cnt = 0; my $blocks_late = 0; my $lost_secs = 0; my @late_msgs = (); foreach $line (@lines) { chomp $line; @arr = split(",",$line); # 0 1 2 3 4 5 6 7 8 9 10 # my $line = "$fid,$callsign,$lat,$lon,$alt_ft,$model,$spd_kts,$hdg,$dist_nm,$upd,$tot_secs\n"; $fid = $arr[0]; next if ($fid eq 'fid'); $callsign = $arr[1]; $lat = $arr[2]; $lon = $arr[3]; $alt_ft = $arr[4]; $model = $arr[5]; $spd_kts = $arr[6]; $hdg = $arr[7]; $dist_nm = $arr[8]; $upd = $arr[9]; $tsecs = $arr[10]; ### $curr_upd = $upd if ($lnn == 0); $lnn++; if ($curr_upd ne $upd) { $block_cnt++; if (length($curr_upd)) { $ee = str2time($upd); $be = str2time($curr_upd); $elap = $ee - $be; if ($elap > $def_min_elap) { $tm = get_time_stg($elap); $blocks_late++; $msg = "$blocks_late:$lnn: Block:$block_cnt: Last: $curr_upd current: $upd - Elapsed $tm"; prt("$msg\n"); push(@late_msgs,$msg); $lost_secs += $elap; } } $curr_upd = $upd; } if (defined $hash{$fid}) { $ra = $hash{$fid}; ${$ra}[7] = $dist_nm; ${$ra}[8] = $tsecs; ${$ra}[11] = $upd; $ra2 = ${$ra}[12]; push(@{$ra2},[$lat,$lon,$alt_ft,$spd_kts,$hdg,$upd]); } else { # 0 1 2 3 4 5 my @a = [$lat,$lon,$alt_ft,$spd_kts,$hdg,$upd]; # 0 1 2 3 4 5 6 7 8 9 10 11 12 $hash{$fid} = [$callsign,$lat,$lon,$alt_ft,$model,$spd_kts,$hdg,$dist_nm,$tsecs,0,$upd,'',\@a]; } } my @fids = keys %hash; @fids = sort mycmp_decend_n @fids; $cnt = scalar @fids; prt("Collected $cnt FID, from $lnn records... $block_cnt json blocks...\n"); my $skipped = 0; my $skip_time = 0; my $would_skip = 0; my $flt_cnt = 0; my ($lat2,$lon2,$alt2,$spd2,$hdg2,$ra3); my ($min_lon,$min_lat,$max_lon,$max_lat); my ($res,$az1,$az2,$dist,$ds,$fid_dist,$fid_time); my $xg = "# Display of $cnt FIDS\n"; my ($msg1,$msg2,$fxg); my $max_cnt = 0; my $min_cnt = 99999; foreach $fid (@fids) { $ra = $hash{$fid}; $callsign = ${$ra}[0]; $lat = ${$ra}[1]; $lon = ${$ra}[2]; $alt_ft = ${$ra}[3]; $model = ${$ra}[4]; $spd_kts = ${$ra}[5]; $hdg = ${$ra}[6]; $dist_nm = ${$ra}[7]; $tsecs = ${$ra}[8]; $upd = ${$ra}[10]; $upd2 = ${$ra}[11]; $be = str2time($upd); $ee = str2time($upd2); $elap = $ee - $be; $tm = get_time_stg($elap); ### $tm .= " (".get_time_stg($tsecs).")"; ######################################## #### Eliminate flights that - #### Are NOT alive for very long if ($elap < $def_min_secs) { $skipped++; $skip_time++; next; } #### Did not move the min distance if ($dist_nm < $def_min_dist) { # but helicopters may not move that much distance, # so keep it anyway, if for a LONGtime - 15 minutes plus if ($elap < $def_min_secs2) { $skipped++; next; } $would_skip++; } $flt_cnt++; # count this FLIGHT $ra2 = ${$ra}[12]; $cnt = scalar @{$ra2}; # record count $min_cnt = $cnt if ($cnt && ($cnt < $min_cnt)); $max_cnt = $cnt if ($cnt > $max_cnt); ###prt("$fid: $callsign $model $lat $lon $alt_ft $hdg $spd_kts\n"); $ds = get_sg_dist_stg( $dist_nm * $NM2KM * 1000 ); ###$msg1 = "$fid: $callsign, $model, $cnt recs, in $tm, $dist_nm nm."; $msg1 = "$fid: $callsign, $model, $cnt recs, in $tm, $ds."; ### prt("$msg1\n"); $fxg = "# $msg1\n"; $min_lon = 400; $min_lat = 400; $max_lon = -400; $max_lat = -400; $fid_dist = 0; $fid_time = 0; $cnt = 0; foreach $ra3 (@{$ra2}) { # 0 1 2 3 4 5 #my @a = [$lat,$lon,$alt_ft,$spd_kts,$hdg,$upd]; $lat2 = ${$ra3}[0]; $lon2 = ${$ra3}[1]; $alt2 = ${$ra3}[2]; $spd2 = ${$ra3}[3]; $hdg2 = ${$ra3}[4]; $upd2 = ${$ra3}[5]; $min_lat = $lat2 if ($lat2 < $min_lat); $min_lon = $lon2 if ($lon2 < $min_lon); $max_lat = $lat2 if ($lat2 > $max_lat); $max_lon = $lon2 if ($lon2 > $max_lon); $ds = "none"; if (($lat != $lat2) || ($lon != $lon2)) { $res = fg_geo_inverse_wgs_84($lat2,$lon2,$lat,$lon,\$az1,\$az2,\$dist); # $mps = $dist / $secs; next if ($dist < $SG_EPSILON); $fid_dist += $dist; $ds = get_sg_dist_stg($fid_dist); } else { next; } $be = str2time($upd); $ee = str2time($upd2); $elap = $ee - $be; $fid_time += $elap; $tm = get_time_stg($fid_time); $cnt++; $fxg .= "$lon2 $lat2 ; $alt2 $spd2 $hdg2 $upd2 $ds $tm\n"; ####### UPDATE ###### $lon = $lon2; $lat = $lat2; $upd = $upd2; ####### DISPLAY ####### if (VERB5()) { # 1469842570000: prt(" ".get_ll_double($lat2).",".get_ll_double($lon2).",". get_alt_stg($alt2).",".get_spd_stg($spd2).",". get_hdg_stg($hdg2).",$upd2,$ds\n"); } } $fxg .= "NEXT\n"; $tm = get_time_stg($fid_time); $ds = get_sg_dist_stg($fid_dist); # 1469848570000: $fid = "Summary "; $msg2 = "$fid: $callsign, $model, $cnt recs, in $tm, $ds."; if (VERB2()) { prt("$msg1\n"); prt("$msg2\n"); } $xg .= "# $msg2\n"; $xg .= $fxg; } $tm = get_time_stg($def_min_secs2); prt("Have $flt_cnt flights, skipped $skipped, where $skip_time were (min.time < $def_min_secs), ". "\nor ".($skipped - $skip_time)." were (dist < $def_min_dist nm and time < $tm), kept $would_skip,...\n"); prt("Json blocks tot $block_cnt - min $min_cnt max $max_cnt \n"); if ($blocks_late) { $tm = get_time_stg($lost_secs); prt("Note: There were $blocks_late block late, ie > $def_min_elap secs, lost $tm\n"); } #my $out_xg = $temp_dir.$PATH_SEP."tempcsv.xg";'; write2file($xg,$out_xg); prt("Written XG to$out_xg\n"); ###$load_log = 1; } ######################################### ### MAIN ### parse_args(@ARGV); process_in_file($in_file); pgm_exit(0,""); ######################################## sub need_arg { my ($arg,@av) = @_; pgm_exit(1,"ERROR: [$arg] must have a following argument!\n") if (!@av); } sub parse_args { my (@av) = @_; my ($arg,$sarg); my $verb = VERB2(); while (@av) { $arg = $av[0]; if ($arg =~ /^-/) { $sarg = substr($arg,1); $sarg = substr($sarg,1) while ($sarg =~ /^-/); if (($sarg =~ /^h/i)||($sarg eq '?')) { give_help(); pgm_exit(0,"Help exit(0)"); } elsif ($sarg =~ /^v/) { if ($sarg =~ /^v.*(\d+)$/) { $verbosity = $1; } else { while ($sarg =~ /^v/) { $verbosity++; $sarg = substr($sarg,1); } } $verb = VERB2(); prt("Verbosity = $verbosity\n") if ($verb); } elsif ($sarg =~ /^l/) { if ($sarg =~ /^ll/) { $load_log = 2; } else { $load_log = 1; } prt("Set to load log at end. ($load_log)\n") if ($verb); } elsif ($sarg =~ /^o/) { need_arg(@av); shift @av; $sarg = $av[0]; $out_file = $sarg; prt("Set out file to [$out_file].\n") if ($verb); } else { pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n"); } } else { $in_file = $arg; prt("Set input to [$in_file]\n") if ($verb); } shift @av; } if ($debug_on) { prtw("WARNING: DEBUG is ON!\n"); if (length($in_file) == 0) { $in_file = $def_file; prt("Set DEFAULT input to [$in_file]\n"); } } if (length($in_file) == 0) { pgm_exit(1,"ERROR: No input files found in command!\n"); } if (! -f $in_file) { pgm_exit(1,"ERROR: Unable to find in file [$in_file]! Check name, location...\n"); } } sub give_help { prt("$pgmname: version $VERS\n"); prt("Usage: $pgmname [options] in-file\n"); prt("Options:\n"); prt(" --help (-h or -?) = This help, and exit 0.\n"); prt(" --verb[n] (-v) = Bump [or set] verbosity. def=$verbosity\n"); prt(" --load (-l) = Load LOG at end. ($outfile)\n"); prt(" --out <file> (-o) = Write output to this file.\n"); } # eof - template.pl