#!/usr/bin/perl -w
# NAME: getcfxml02.pl
# AIM: Fetch the crossfeed flights.xml, and keep some flight stats, using XML:Simple
# 2018-03-06 - Review and make some changes, add seconds, verbosity, out dir, etc...
use strict;
use warnings;
use File::Basename;  # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] )
use File::stat;
use XML::Simple;
use Data::Dumper;
use LWP::Simple;
use JSON;
use Cwd;
my $os = $^O;
my $perl_dir = '/home/geoff/bin';
my $PATH_SEP = '/';
my $temp_dir = '/tmp';
if ($os =~ /win/i) {
    $perl_dir = 'C:\GTools\perl';
    $temp_dir = $perl_dir;
    $PATH_SEP = "\\";
}
unshift(@INC, $perl_dir);
require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl' Check paths in \@INC...\n";
# log file stuff
our ($LF);
my $pgmname = $0;
if ($pgmname =~ /(\\|\/)/) {
    my @tmpsp = split(/(\\|\/)/,$pgmname);
    $pgmname = $tmpsp[-1];
}
my $outfile = $temp_dir.$PATH_SEP."temp.$pgmname.txt";
open_log($outfile);

# user variables
my $VERS = "0.0.2 2018-03-06";
#my $VERS = "0.0.1 2013-04-30";
my $load_log = 0;
my $in_file = '';
my $verbosity = 1;
my $out_dir = $perl_dir;
my $json_dir = $out_dir.$PATH_SEP.'json'; # use <root>/json as output
my $out_file = $json_dir.$PATH_SEP.'usage01.json';
my $out_file2 = $json_dir.$PATH_SEP.'usage01.txt';
my $url = 'http://crossfeed.freeflightsim.org/flights.xml';
###my $url = 'http://crossfeed.fgx.ch/flights.xml';
my $oldest_time = time();
my $sleep_secs = 10;
# ### DEBUG ###
my $debug_on = 0;
#my $def_file = 'def_file';
my $def_file = $temp_dir.$PATH_SEP."tempcfxml.txt";;
my $def_out = $temp_dir.$PATH_SEP.'tempdef.json';

### program variables
my @warnings = ();
my $cwd = cwd();
my $max_cs = 0;
my $max_mod = 0;
my %model_hash = ();
my %callsign_hash = ();
sub write_ref_hashes($);


sub VERB1() { return $verbosity >= 1; }
sub VERB2() { return $verbosity >= 2; }
sub VERB5() { return $verbosity >= 5; }
sub VERB9() { return $verbosity >= 9; }

sub show_warnings($) {
    my ($val) = @_;
    if (@warnings) {
        prt( "\nGot ".scalar @warnings." WARNINGS...\n" );
        foreach my $itm (@warnings) {
           prt("$itm\n");
        }
        prt("\n");
    } else {
        prt( "\nNo warnings issued.\n\n" ) if (VERB9());
    }
}

sub pgm_exit($$) {
    my ($val,$msg) = @_;
    if (length($msg)) {
        $msg .= "\n" if (!($msg =~ /\n$/));
        prt($msg);
    }
    show_warnings($val);
    close_log($outfile,$load_log);
    exit($val);
}


sub prtw($) {
   my ($tx) = shift;
   $tx =~ s/\n$//;
   prt("$tx\n");
   push(@warnings,$tx);
}

# RENAME A FILE TO .1, .2, .3...
sub rename_2_nxt_num {
	my ($fil) = shift;
	my $ret = 0;	# assume NO SUCH FILE
	if ( -f $fil ) {	# is there?
        $ret = 1;
        my $nfil = $fil.".$ret";
        while (-f $nfil) {
            $ret++;
            $nfil = $fil.".$ret";
        }
		rename $fil, $nfil;
	}
	return $ret;
}

sub get_oldest_time_in_dir($) {
    my $dir = shift;
    if (!opendir(DIR,$dir)) {
        return time();
    }
    my @files = readdir(DIR);
    closedir(DIR);
    ut_fix_directory(\$dir);
    my ($file,$ff,$sb);
    my $tm = time();
    foreach $file (@files) {
        next if ($file eq '.');
        next if ($file eq '..');
        $ff = $dir.$file;
        next if (-d $ff); # skip directories
        if ($sb = stat($ff)) {
            $tm = $sb->mtime if ($sb->mtime < $tm);
        }
    }
    return $tm;
}

#Processing  74 lines, from [tempusage.json]...
#Dump of perl scalar...
#$VAR1 = {
#          'cs_cnt' => 41,
#          'models' => [
#                        {
#                          'usr_cnt' => 1,
#                          'model' => '717-200',
#                          'users' => [
#                                       'dedokun'
#                                     ]
#                        },
#                      ],
#          'success' => bless( do{\(my $o = 1)}, 'JSON::XS::Boolean' ),
#          'callsigns' => [
#                           {
#                             'models' => [
#                                           '777-200'
#                                         ],
#                             'callsign' => 'ANA108',
#                             'mod_cnt' => 1
#                           },
#                         ],
#          'updated' => '2013/04/30 12:23:45 UTC',
#          'mod_cnt' => 28,
#          'generator' => 'xmlsimple02.pl'
#        };
#
#End DUMP
sub reload_previous_json($) {
    my $inf = shift;
    if (! open INF, "<$inf") {
       my ($name,$dir) = fileparse($inf);
       if (-d $dir) {
           prtw("WARNING: Unable to open file [$inf]. Assume none...\n"); 
       } else {
           prt("WARNING: Unable to open file [$inf]...\n"); 
           pgm_exit(1,"ERROR: Unable to stat dir [$dir]\n"); 
       }
       return;
    }
    my @lines = <INF>;
    close INF;
    my $lncnt = scalar @lines;
    prt("Processing $lncnt lines, from [$inf]...\n");
    my $line = join(" ",@lines);
    my $json = JSON->new->allow_nonref;
    my $rh = $json->decode( $line );
    ###my $tmp = Dumper($rh);
    my ($ra,$cnt,$mc,$cc,$mh,$mod,$rua,$user,$refha,$cs,$len);
    $mc = 0;
    $cc = 0;
    if (defined ${$rh}{'mod_cnt'}) {
        $mc = ${$rh}{'mod_cnt'};
    }
    if (defined ${$rh}{'cs_cnt'}) {
        $cc = ${$rh}{'cs_cnt'};
    }
    if (defined ${$rh}{'models'}) {
        $ra = ${$rh}{'models'};
        $cnt = scalar @{$ra};
        prt("Got $cnt model hashes...($mc)\n");
        foreach $mh (@{$ra}) {
            # 'usr_cnt' => 1,
            # 'model' => '717-200',
            # 'users' => ['dedokun']
            if ((defined ${$mh}{'model'})&& (defined ${$mh}{'users'})) {
                $mod = ${$mh}{'model'};
                $len = length($mod);
                $max_mod = $len if ($len > $max_mod);
                $rua = ${$mh}{'users'};
                $model_hash{$mod} = {} if (!defined $model_hash{$mod});
                $refha = $model_hash{$mod};
                foreach $cs (@{$rua}) {
                    if (defined ${$refha}{$cs}) {
                        ${$refha}{$cs}++;
                    } else {
                        ${$refha}{$cs} = 1;
                        $len = length($cs);
                        $max_cs = $len if ($len > $max_cs);
                    }
                }
            } else {
                prt(Dumper($mh));
                pmg_exit(2,"ERROR: 'model' or 'users' NOT defined!\n");
            }
        }
    }
    if (defined ${$rh}{'callsigns'}) {
        $ra = ${$rh}{'callsigns'};
        $cnt = scalar @{$ra};
        prt("Got $cnt callsign hashes...($cc)\n");
        # 'models' => ['777-200'],
        # 'callsign' => 'ANA108',
        # 'mod_cnt' => 1
        foreach $mh (@{$ra}) {
            if ((defined ${$mh}{'callsign'})&& (defined ${$mh}{'models'})) {
                $cs  = ${$mh}{'callsign'};
                $rua = ${$mh}{'models'};
                $callsign_hash{$cs} = {} if (!defined $callsign_hash{$cs});
                $refha = $callsign_hash{$cs};
                foreach $user (@{$rua}) {
                    if (defined ${$refha}{$user}) {
                        ${$refha}{$user}++;
                    } else {
                        ${$refha}{$user} = 1;
                    }
                }
            } else {
                prt(Dumper($mh));
                pmg_exit(2,"ERROR: 'callsign' or 'users' NOT defined!\n");
            }
        }
    }
    rename_2_nxt_num($inf); # get it out of the way for the next
    write_ref_hashes($def_out) if (VERB9());
}


################################################
### json format
# Characters '"', '\', and '/' are escaped with '\'
sub get_json($) {
    my $txt = shift;
    my $len = length($txt);
    my ($i,$ch);
    my $json = '';
    for ($i = 0; $i < $len; $i++) {
        $ch = substr($txt,$i,1);
        $json .= "\\" if (($ch eq '"')||($ch eq "\\")||($ch eq '/')) ;
        $json .= $ch;
    }
    return $json;
}

sub mycmp_nc_sort {
   return -1 if (lc($a) lt lc($b));
   return 1 if (lc($a) gt lc($b));
   return 0;
}

sub mycmp_ascend_n1 {
   return -1 if (${$a}[1] < ${$b}[1]);
   return 1 if (${$a}[1] > ${$b}[1]);
   return 0;
}
sub mycmp_decend_n1 {
   return -1 if (${$a}[1] > ${$b}[1]);
   return 1 if (${$a}[1] < ${$b}[1]);
   return 0;
}

sub get_hash_list($) {
    my $rh = shift;
    my $msg = '';
    my ($key,$val);
    my @arr = sort mycmp_nc_sort keys(%{$rh});
    foreach $key (@arr) {
        $val = ${$rh}{$key};
        $msg .= "," if (length($msg));
        $msg .= "$key=$val";
    }
    return $msg;
}

sub out_usage_list($$) {
    my ($ra,$msg) = @_; # \@usage,$msg
    my ($val,$mod,$cnt,$tcnt,@arr,$tmp,$acnt);
    # combine some models
    my %h = ();
    $tcnt = 0;
    my $mod_777 = '777-*';
    my $mod_atc = 'ATC-*';
    my %mod_hash = ();
    my %atc_hash = ();
    foreach $val (@{$ra}) {
        $tcnt++;
        $mod = ${$val}[0];
        $cnt = ${$val}[1];
        if ($mod =~ /^777-/) {
            if (defined $mod_hash{$mod}) {
                $mod_hash{$mod} += $cnt;
            } else {
                $mod_hash{$mod} = $cnt;
            }
            $mod = $mod_777; # '777-*';
        } elsif (($mod =~ /atc/i)||($mod =~ /OpenRadar/i)) {
            if (defined $atc_hash{$mod}) {
                $atc_hash{$mod} += $cnt;
            } else {
                $atc_hash{$mod} = $cnt;
            }
            $mod = $mod_atc; # 'ATC-*';
        }
        if (defined $h{$mod}) {
            $h{$mod} += $cnt;
        } else {
            $h{$mod} = $cnt;
        }
    }
    my @usage = ();
    my $total = 0;
    @arr = sort mycmp_nc_sort keys(%h);
    foreach $mod (@arr) {
        $cnt = $h{$mod};
        push(@usage,[$mod,$cnt]);
        $total += $cnt;
    }
    @usage = sort mycmp_decend_n1 @usage;
    $cnt = scalar @usage;
    my $curr = time();
    my $diff = $curr - $oldest_time;
    my $rt = secs_HHMMSS($diff);
    my $txt = "# Usage list, from ".lu_get_YYYYMMDD_hhmmss_UTC($oldest_time)." UTC, now at ".lu_get_YYYYMMDD_hhmmss_UTC($curr)." UTC\n";
    $txt .= "# Total of $cnt model groups, $tcnt total, with $total callsigns, over $rt\n";
    foreach $val (@usage) {
        $mod = ${$val}[0];
        $cnt = ${$val}[1];
        $tmp = '';
        if ($mod eq $mod_777) {
            $tmp = get_hash_list(\%mod_hash);
        } elsif ($mod eq $mod_atc) {
            $tmp = get_hash_list(\%atc_hash);
        }
        $mod .= ' ' while (length($mod) < $max_mod);
        $acnt = sprintf("%3d",$cnt);
        $txt .= "$mod $acnt $tmp\n";
    }
    $txt .= "# eof $out_file2\n";
    write2file($txt,$out_file2);
    prt("$rt $msg\n") if (VERB1());
}

sub write_ref_hashes($) {
    my $out = shift;
    my ($key,$val,$key2,$val2,$rh,$json,@arr,$cnt,@arr2,$cnt2,$msg);
    $rh = \%model_hash;
    my @usage = ();
    # per model
    @arr2 = sort mycmp_nc_sort keys(%{$rh});
    $cnt = scalar @arr2;
    $json = '{"success":true,"generator":"'.$pgmname.'","updated":"'.lu_get_YYYYMMDD_hhmmss_UTC(time()).' UTC",'."\n";
    $json .= '"mod_cnt":'.$cnt;
    $json .= ',"models":['."\n";
    $msg = "Model count $cnt";
    $cnt2 = 0;
    foreach $key (@arr2) {
        $val = ${$rh}{$key};
        @arr = sort keys(%{$val});
        $cnt = scalar @arr;
        $json .= ",\n" if ($cnt2);
        $json .= '{"model":"'.get_json($key).'","usr_cnt":'.$cnt.',"users":[';
        push(@usage, [$key,$cnt]);
        $cnt = 0;
        foreach $key2 (@arr) {
            $val2 = ${$val}{$key2}; # get count by pilot
            $json .= ',' if ($cnt);
            $json .= '"'.get_json($key2).'"';
            $cnt++;
        }
        $json .= ']}';
        $cnt2++;
    }
    $json .= "\n";
    $json .= '],'."\n";
    # per callsign
    $rh = \%callsign_hash;
    @arr2 = sort  mycmp_nc_sort keys(%{$rh});
    $cnt = scalar @arr2;
    $json .= '"cs_cnt":'.$cnt;
    $json .= ',"callsigns":['."\n";
    $msg .= ", callsign count $cnt";
    $cnt2 = 0;
    foreach $key (@arr2) {
        $val = ${$rh}{$key};
        @arr = sort keys(%{$val});
        $cnt = scalar @arr;
        $json .= ",\n" if ($cnt2);
        $json .= '{"callsign":"'.get_json($key).'","mod_cnt":'.$cnt.',"models":[';
        $cnt = 0;
        foreach $key2 (@arr) {
            $val2 = ${$val}{$key2}; # get count by pilot
            $json .= ',' if ($cnt);
            $json .= '"'.get_json($key2).'"';
            $cnt++;
        }
        $json .= ']}';
        $cnt2++;
    }
    $json .= "\n";
    $json .= ']}'."\n";
    prt($json) if (VERB9());
    write2file($json,$out);
    $msg .= ", to [$out]";
    ##prt("$msg, to [$out]\n");
    # out a USAGE list
    out_usage_list(\@usage,$msg);
}

sub process_in_file($) {
    my ($inf) = @_;
    prt("Processing [$inf]...\n") if (VERB9());
    my $xml = XMLin($inf);
    if (defined ${$xml}{'marker'}) {
        my $ra = ${$xml}{'marker'};
        #prt("Dump of ref array...\n");
        #prt(Dumper($ra));
        my ($rh,$cs,$mod,$refha,$len);
        prt("Dump of each hash...or callsign and model strings\n") if (VERB9());
        #'alt' => '102',
        #'callsign' => 'gagaga',
        #'lat' => '36.714629',
        #'model' => 'v22',
        #'heading' => '8',
        #'spd_kt' => '0',
        #'lng' => '126.492990',
        #'server_ip' => '203.251.5.253'
        foreach $rh (@{$ra}) {
            if ( (defined ${$rh}{'callsign'}) && (defined ${$rh}{'model'}) ) {
                $cs = ${$rh}{'callsign'};
                $mod = ${$rh}{'model'};
                prt("$mod $cs\n") if (VERB9());
                $model_hash{$mod} = {} if (!defined $model_hash{$mod});
                $refha = $model_hash{$mod};
                if (defined ${$refha}{$cs}) {
                    ${$refha}{$cs}++;
                } else {
                    ${$refha}{$cs} = 1;
                    $len = length($cs);
                    $max_cs = $len if ($len > $max_cs);
                }
                $callsign_hash{$cs} = {} if (!defined $callsign_hash{$cs});
                $refha = $callsign_hash{$cs};
                if (defined ${$refha}{$mod}) {
                    ${$refha}{$mod}++;
                } else {
                    ${$refha}{$mod} = 1;
                    $len = length($mod);
                    $max_mod = $len if ($len > $max_mod);
                }
            } else {
                prt(Dumper($rh));
            }
        }
    } else {
        prt("Dump of WHOLE\n");
        prt(Dumper($xml));
    }
    #my $xml2 = XMLin($inf, KeyAttr => {marker => 'model'});
    #prt(Dumper($xml2));
    #$load_log = 1;
    write_ref_hashes($out_file);
}

sub get_crossfeed() {
    my $cf = get($url);
    process_in_file($cf);
}

#########################################
### MAIN ###
parse_args(@ARGV);
### pgm_exit(1,"");
$oldest_time = get_oldest_time_in_dir($json_dir);
reload_previous_json($out_file);
while (1) {
    get_crossfeed();
    sleep($sleep_secs); # default was 10
}

#process_in_file($in_file);
pgm_exit(0,"");
########################################

sub need_arg {
    my ($arg,@av) = @_;
    pgm_exit(1,"ERROR: [$arg] must have a following argument!\n") if (!@av);
}

sub parse_args {
    my (@av) = @_;
    my ($arg,$sarg);
    while (@av) {
        $arg = $av[0];
        if ($arg =~ /^-/) {
            $sarg = substr($arg,1);
            $sarg = substr($sarg,1) while ($sarg =~ /^-/);
            if (($sarg =~ /^h/i)||($sarg eq '?')) {
                give_help();
                pgm_exit(0," Help exit(0)");
            } elsif ($sarg =~ /^v/) {
                if ($sarg =~ /^v.*(\d+)$/) {
                    $verbosity = $1;
                } else {
                    while ($sarg =~ /^v/) {
                        $verbosity++;
                        $sarg = substr($sarg,1);
                    }
                }
                prt("Verbosity = $verbosity\n") if (VERB1());
            #} elsif ($sarg =~ /^l/) {
            #    if ($sarg =~ /^ll/) {
            #        $load_log = 2;
            #    } else {
            #        $load_log = 1;
            #    }
            #    prt("Set to load log at end. ($load_log)\n") if (VERB1());
            } elsif ($sarg =~ /^o/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                $out_dir = $sarg;
                if (-d $out_dir) {
                    $json_dir = $out_dir; # use user dir as output
                    $out_file = $json_dir.$PATH_SEP.'usage01.json';
                    $out_file2 = $json_dir.$PATH_SEP.'usage01.txt';
                    prt("Set output dir to [$out_dir].\n") if (VERB1());
                } else {
                    prt("Error: Can NOT find dir '$out_dir'!\n");
                    pgm_exit(1, "Output dir MUST exist. Will NOT be created`\n");
                }
            } elsif ($sarg =~ /^s/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                if ($sarg =~ /^\d+$/) {
                    if ($sarg == 0) {
                        pgm_exit(1,"ERROR: Invalid argument [$arg $sarg]! Min is 1\n");
                    }
                    $sleep_secs = $sarg;
                } else {
                    prt("Seconds MUST be an integer, not $sarg\n");
                    pgm_exit(1,"ERROR: Invalid argument [$arg $sarg]! Try -?\n");
                }
            } else {
                pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n");
            }
        } else {
            pgm_exit(1,"ERROR: Invalid argument [$arg]! Try -?\n");
            #$in_file = $arg;
            #prt("Set input to [$in_file]\n") if (VERB1());
        }
        shift @av;
    }

    #if ($debug_on) {
    #    prtw("WARNING: DEBUG is ON!\n");
    #    if ((length($in_file) ==  0) && $debug_on) {
    #        $in_file = $def_file;
    #        prt("Set DEFAULT input to [$in_file]\n");
    #    }
    #}
    #if (length($in_file) ==  0) {
    #    pgm_exit(1,"ERROR: No input files found in command!\n");
    #}
    #if (! -f $in_file) {
    #    pgm_exit(1,"ERROR: Unable to find in file [$in_file]! Check name, location...\n");
    #}
    #$verbosity = 9;
}

sub give_help {
    prt("$pgmname: version $VERS\n");
    prt("Usage: $pgmname [options]\n");
    prt("Options:\n");
    prt(" --help  (-h or -?) = This help, and exit 0.\n");
    prt(" --verb[n]     (-v) = Bump [or set] verbosity. def=$verbosity\n");
    #prt(" --load        (-l) = Load LOG at end. ($outfile)\n");
    prt(" --out <dir>   (-o) = Write outputs to this dir. (def=$json_dir)\n");
    prt(" --secs <sec>  (-s) = An integer num of seconds to sleep between fetches. (def=$sleep_secs)\n");
    prt("\n");
    prt(" If not 'help' then will commence to fetch the crossfeed xml each $sleep_secs secs\n");
    prt(" from this $url,\n");
    prt(" and process it to extract aircraft and pilots, and keep an updated\n");
    prt(" list output to $out_file2 until stopped, ctrl+c...\n");
    prt(" The list with more details is also written to $out_file.\n");
    prt(" In other words gathers MP aircraft usage using the crossfeed xml.\n");
}

# eof - getcfxml02.pl