#!/perl -w
# NAME: cleandiff02.pl
# AIM: To remove some things from an input diff file ...
# 1 - remove Only in blah blah ...
# 2 - remove diff of .dsw, .dsp, .sln, .vcproj files ...
# and write results to an output file
# 01/02/2015 - Add -i input.txt, similar to -@input.txt
# 02/12/2013 - Add -x *.html;*.sip list of wild card excludes
# 22/09/2013 - Add a -KO to add ALL 'Only in ...' lines
# 05/09/2013 - Modify -ko options to NOT show .git folder items, and order the list
# 11/02/2013 - Default output changed to tempclean.diff
# 28/03/2012 - Add .vcxproj to excluded list
# 23/03/2012 - Add -u to convert paths to unix path speparator
# 05/01/2012 - Added special -x _cmake_, and excluded file '.gitignore'
# 10/12/2011 - If VERB, output a changed file list
# 17/11/2011 - Add -o <file>, to set the OUTPUT of the cleaned diff
# 29/06/2011 - Add -x <file>, to exclude a specific 'file' from the diff
# 26/05/2011 - Improve UI
use strict;
use warnings;
use File::Basename;  # split path ($name,$dir,$ext) = fileparse($file [, qr/\.[^.]*/] )
use Cwd;
my $perl_dir = 'C:\GTools\perl';
unshift(@INC, $perl_dir);
require 'lib_utils.pl' or die "Unable to load 'lib_utils.pl'!\n";
# log file stuff
our ($LF);
my $pgmname = $0;
if ($pgmname =~ /(\\|\/)/) {
    my @tmpsp = split(/(\\|\/)/,$pgmname);
    $pgmname = $tmpsp[-1];
}
my $outfile = $perl_dir."\\temp.$pgmname.txt";
open_log($outfile);

my $VERS = "1.0.5 2013-09-22";

# features
my $load_log = 0;   # load log file at end
my $keep_only_in = 0;
my $KEEP_ONLY_IN = 0;
my $keep_bin_files = 0;
my $unix_pat_sep = 0;
my $include_msvc = 0;
# program variables
my $in_file  = '';
my $out_file = $perl_dir.'\tempclean.diff';
my @user_xcludes = ();
my @xclude_dirs = ();
my $verbosity = 0;
my $in_input_file = 0;
my $show_copy_list = 0;
my $output_file = '';
my $add_skipped_lines = 0;

my $common_destination = '';    # from $fil2
my $common_source = '';         # from $fil1

my @changed_files = ();
my @excluded_files = ();
my @new_files = ();     # that is '@@ -0,0 +n,n @@' COUNT LINE
my @excluded_new_files = ();    # likewise NEW, but excluded
my @cmake_xcludes = qw( CMakeLists.txt *.cmake );

sub VERB1() { return ($verbosity >= 1); }
sub VERB2() { return ($verbosity >= 2); }
sub VERB5() { return ($verbosity >= 5); }
sub VERB9() { return ($verbosity >= 9); }

sub get_common_dir($$) {
    my ($file,$rdir) = @_;
    my $len1 = length($file);
    return 0 if ($len1 == 0);
    my $dir2 = ${$rdir};
    my $len2 = length($dir2);
    my ($name,$dir1) = fileparse($file);
    return 0 if ($dir1 =~ /^\.(\\|\/)$/);
    $len1 = length($dir1);
    if ($len2 == 0) {
        ${$rdir} = $dir1;   # set first common directory
        return 1;
    }
    # else keep only the COMMON part of $dir2 given in $dir1
    my $len = ($len2 < $len1) ? $len2 : $len1;  # get shortest
    my ($i,$ch1,$ch2);
    for ($i = 0; $i < $len; $i++) {
        $ch1 = substr($dir1,$i,1);
        $ch2 = substr($dir2,$i,1);
        last if ($ch1 ne $ch2);
    }
    if ($i < $len2) {
        $dir2 = substr($dir2,0,$i);
        ${$rdir} = $dir2;
    }
    return 1;
}

sub get_common_dst($) {
    my $fil = shift;
    get_common_dir($fil,\$common_destination);
}

sub get_common_src($) {
    my $fil = shift;
    get_common_dir($fil,\$common_source);
}

my $dbg_mww = 0;
sub match_with_wild($$) {
    my ($fil1,$fil2) = @_;
    my $len1 = length($fil1);
    my $len2 = length($fil2);
    prt("match_with_wild: [$fil1] [$fil2] ") if ($dbg_mww);
    my ($i,$j,$c1,$c2);
    $i = 0;
    $j = 0;
    if (($len1 > 0) && ($len2 > 0)) {
        while (($i < $len1)&&($j < $len2)) {
            $c1 = substr($fil1,$i,1);
            $c2 = substr($fil2,$j,1);
            if (($c1 eq $c2)||($c1 eq '?')||($c2 eq '?')) {
                $i++;
                $j++;
                prt("$c1= ") if ($dbg_mww);
            } elsif ($c2 eq '*') {
                $i++; # any $c1 matches asterick
                if (($j + 1) < $len2) {
                    # but if more, maybe time to step past '*'
                    $c2 = substr($fil2,($j+1),1);
                    if ($c1 eq $c2) {
                        $j += 2;
                    }
                }
                prt("$c1* ") if ($dbg_mww);
            } elsif ($c1 eq '*') {
                $j++; # any $c2 matches asterick
                if (($i + 1) < $len1) {
                    # but if more, maybe time to step past '*'
                    $c1 = substr($fil1,($i+1),1);
                    if ($c1 eq $c2) {
                        $i += 2;
                    }
                }
                prt("$c2* ") if ($dbg_mww);
            } else {
                prt(" = 0 - [$c1] ne [$c2]\n") if ($dbg_mww);
                return 0;
            }
        }
        if (($i == $len1)&&($j == $len2)) {
            prt(" = 1 - both ran out of chars\n") if ($dbg_mww);
            return 1; # both ran out of chars
        } elsif (($i == $len1)&&($c2 eq '*')&&(($j + 1) == $len2)){
            prt(" = 1 - first ran out and last is second $c2\n") if ($dbg_mww);
            return 1; # first ran out, and second is last '*'
        } elsif (($j == $len2)&&($c1 eq '*')&&(($i + 1) == $len1)){
            prt(" = 1 - second ran out and last of first is $c1\n") if ($dbg_mww);
            return 1; # second ran out, and second is last '*'
        } 
        prt(" = 0 - failed - no case\n") if ($dbg_mww);
    } elsif ($len1 > 0) {
        # 2nd is nul
        if ($fil1 eq '*') {
            prt(" = 1 - asterix matches nul\n") if ($dbg_mww);
            return 1; # nul matches asterix
        }
        prt(" = 0 - len1 > 0, but [$fil1]\n") if ($dbg_mww);
    } elsif ($len2 > 0) {
        # 1st is nul
        if ($fil2 eq '*') {
            prt(" = 1 - nul match asterix\n") if ($dbg_mww);
            return 1; # nul matches asterix
        }
        prt(" = 0 - len2 > 0, but [$fil1]\n") if ($dbg_mww);
    } else {
        prt(" = 0 - no case\n") if ($dbg_mww);
    }
    return 0;
}

sub matches_wild($$) {
    my ($fil,$wild) = @_;
	my ($n1,$d1,$e1) = fileparse( $fil, qr/\.[^.]*/ );
	my ($n2,$d2,$e2) = fileparse( $wild, qr/\.[^.]*/ );
    my $lcn1 = lc($n1);
    my $lcn2 = lc($n2);
    # strip . from extension
    $e1 =~ s/^\.//;
    $e2 =~ s/^\.//;
    my $lce1 = lc($e1);
    my $lce2 = lc($e2);
    prt("[v9] matches_wild: [$n1] [$n2] and [$e1] [$e2]\n") if (VERB9());
    return 1 if (($lcn1 eq $lcn2)&&($lce1 eq $lce2));
    return 2 if (($lcn1 eq $lcn2)&&($lce2 eq '*'));
    return 3 if (($lcn2 eq '*')&&($lce1 eq $lce2));
    return 4 if (match_with_wild($lcn1,$lcn2) && match_with_wild($lce1,$lce2));
    return 0;
}

sub has_wild($) {
    my $txt = shift;
    my $len = length($txt);
    my ($i,$c);
    for ($i = 0; $i < $len; $i++) {
        $c = substr($txt,$i,1);
        if (($c eq '?')||($c eq '*')) {
            return 1;
        }
    }
    return 0;
}

sub in_user_excludes($) {
    my ($fil) = shift;
    my ($nam,$dir) = fileparse($fil);
    my ($item);
    my $hw = has_wild($fil);
    foreach $item (@user_xcludes) {
        return 1 if ($nam eq $item);
        if ($hw || has_wild($item)) {
            return 2 if (matches_wild($item,$nam));
        }
    }
    return 0;
}

sub in_dir_excludes($) {
    my ($tx) = @_;
    my ($dir);
    foreach $dir (@xclude_dirs) {
        if ($tx =~ /(\\|\/)$dir(\\|\/)/) {
            return 1;
        }
    }
    return 0;
}

# debug
my $dbg_01 = 0; # show every line
my @warnings = ();

sub prtw($) {
    my $txt = shift;
    $txt =~ s/\n$//;
    prt("$txt\n");
    push(@warnings,$txt);
}

sub show_warnings($) {
    my ($dbg) = shift;
    if (@warnings) {
        prt( "\nGot ".scalar @warnings." WARNINGS ...\n" );
        foreach my $line (@warnings) {
            prt("$line\n" );
        }
        prt("\n");
    } elsif ($dbg) {
        #prt("\nNo warnings issued.\n\n");
    }
}


sub pgm_exit($$) {
    my ($val,$msg) = @_;
    show_warnings($val);
    prt( "$msg" ) if (length($msg));
    close_log($outfile,$load_log);
    exit($val);
}

sub is_msvc_file($) {
    my ($tx) = @_;
    my $val = 0;
    # exclude MSVC make files
    $val++;
    return $val if ($tx =~ /\.dsp$/i);      # 1
    $val++;
    return $val if ($tx =~ /\.dsw$/i);
    $val++;
    return $val if ($tx =~ /\.sln$/i);
    $val++;
    return $val if ($tx =~ /\.vcproj$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj.user$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj.filters$/i);  # 6
    return 0;
}

my $repo_dirs = 3;
sub is_excluded_file($) {
    my ($tx) = @_;
    my $val = 0;
    # exclude SVN, CVS, git directory items
    $val++;
    return $val if ($tx =~ /(\\|\/)\.svn(\\|\/)/);
    $val++;
    return $val if ($tx =~ /(\\|\/)CVS(\\|\/)/);
    $val++;
    return $val if ($tx =~ /(\\|\/)\.git(\\|\/)/);
    # exclude MSVC make files, if NOT specifically INCLUDED -m
    return 0 if ($include_msvc && is_msvc_file($tx));
    $val++;
    return $val if ($tx =~ /\.dsp$/i);      # 1
    $val++;
    return $val if ($tx =~ /\.dsw$/i);
    $val++;
    return $val if ($tx =~ /\.sln$/i);
    $val++;
    return $val if ($tx =~ /\.vcproj$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj.user$/i);
    $val++;
    return $val if ($tx =~ /\.vcxproj.filters$/i);  # 6
    # user file exclude
    $val++;
    return $val if (in_user_excludes($tx));
    # user directory exclude
    $val++;
    return $val if (in_dir_excludes($tx));
    $val++;
    return $val if ($tx =~ /\.gitignore$/);
    return 0;
}

sub is_repo_dir($) {
    my $tx = shift;
    return 1 if ($tx =~ /(\\|\/)\.git(\\|\/|:)/);   # Only in lines can be 'fgcom\.git:...'
    return 1 if ($tx =~ /(\\|\/)\.svn(\\|\/)/);
    return 1 if ($tx =~ /(\\|\/)CVS(\\|\/)/);
    return 0;
}


sub get_diff_files($$$) {
    my ($lin,$rfil1,$rfil2) = @_;
    #my @arr = split(/s+/,$lin);
    my @arr = space_split($lin);
    if (scalar @arr < 2) {
        pgm_exit(1,"ERROR: Line [$lin] did NOT split correctly!\n");
    }
    my ($fil,$cnt);
    $cnt = 0;
    foreach $fil (@arr) {
        next if ($fil =~ /^-/);
        if ($cnt == 0) {
            ${$rfil1} = $fil;
            $cnt++;
        } elsif ($cnt == 1) {
            ${$rfil2} = $fil;
            $cnt++;
        }
    }
    return $cnt;
}

# get_counts($lcnts,\$off1,\$len1,\$off2,\$len2);
# Normally  @@ -57,10 +57,16 
# Also must resolve @@ -1 +1 @@ and @@ -0,0 +1 @@
# UGH and '-1 +0,0', '-1,2 +1', '-1 +0,0'
sub get_counts($$$$$) {
    my ($lcnts,$roff1,$rlen1,$roff2,$rlen2) = @_;
    ${$roff1} = -9999;
    ${$rlen1} = -9999;
    ${$roff2} = -9999;
    ${$rlen2} = -9999;
    my ($cnt,$old,$new,$msg,$i);
    my @arr = split(/\s+/,$lcnts);
    $cnt = scalar @arr;
    if ($cnt != 2) {
        $msg = "WARNING: CNTS [$lcnts] did NOT split in 2 on space! Got $cnt?";
        for ($i = 0; $i < $cnt; $i++) {
            $msg .= " $arr[$i]";
        }
        prtw("$msg\n");
        return 1;
    }
    $old = $arr[0];
    $new = $arr[1];
    if ( !($old =~ /^-/) || !($new =~ /^\+/)) {
        prtw("WARNING: CNTS [$lcnts] did NOT resolve with - or + [$old] [$new]!\n");
        return 2;
    }
    $old = substr($old,1);
    @arr = split(",",$old);
    $cnt = scalar @arr;
    if ($cnt == 1) {
        ${$roff1} = $arr[0];
        ${$rlen1} = $arr[0];
    } elsif ($cnt == 2) {
        ${$roff1} = $arr[0];
        ${$rlen1} = $arr[1];
    } else {
        prtw("WARNING: CNTS [$lcnts] OLD did NOT resolve [$old] [$new]!\n");
        return 3;
    }
    $new = substr($new,1);
    @arr = split(",",$new);
    $cnt = scalar @arr;
    if ($cnt == 1) {
        ${$roff2} = $arr[0];
        ${$rlen2} = $arr[0];
    } elsif ($cnt == 2) {
        ${$roff2} = $arr[0];
        ${$rlen2} = $arr[1];
    } else {
        prtw("WARNING: CNTS [$lcnts] OLD did NOT resolve [$old] [$new]!\n");
        return 4;
    }
    return 0;
}

sub process_file($) {
	my ($fil) = @_;
	my (@lines, $line, $lncnt, $i, $txt, $lnnum);
	my @nlines = ();
    my @only = ();
    my @bins = ();
    my @skipped = ();
    my %dupes = ();
	my $newlns = 0;
    my ($res,$proj,$fil1,$fil2,$sline,$cnt);
    my ($name,$dir);
    my ($tmp,$lcnts,$extra,$off1,$len1,$off2,$len2);
    my ($poff1,$plen1,$poff2,$plen2);
    $proj = '';
    $off1 = $len1 = $off2 = $len2 = 0;
    $fil1 = '';
    $fil2 = '';
    if (open INF, "<$fil") {
		@lines = <INF>;
		close INF;
		$lncnt = scalar @lines;
		prt( "Process $lncnt lines from $fil ...\n" );
		$lnnum = 0;
		for ($i = 0; $i < $lncnt; $i++) {
			$line = $lines[$i];
			chomp $line;
			$lnnum = $i + 1;
            prt("$lnnum: [$line]\n") if ($dbg_01);
            if ($line =~ /^diff\s+.+/) {
			#if ( $line =~ /^diff\s+(.+)\s+(.+)$/ ) {
                $sline = $line;
                $sline =~ s/^diff\s+//;
                $fil1 = '';
                $fil2 = '';
                $cnt = get_diff_files($sline,\$fil1,\$fil2);
                if (($cnt != 2)||(length($fil1)==0)) {
                   prtw("WARNING: get_diff_files:1: FAILED on [$sline] $cnt f1=[$fil1] f2=[$fil2]\n");
                }
                get_common_dst($fil1);
                get_common_src($fil2);
                #$res = is_excluded_file($line);
                $res = is_excluded_file($fil1);
                push(@excluded_files,[$fil1,$fil2,$res]) if ($res);
                prt("[v5]$lnnum: $line [$fil1] ($res) 1\n") if (VERB5());
                if (length($fil1) && !defined $dupes{$fil1}) {
                    ($name,$dir) = fileparse($fil1);
                    #                    0     1     2     3    4
                    push(@changed_files,[$name,$fil1,$fil2,$res,0]);
                    $dupes{$fil1} = 1;
                }
                if ($res) {
                    $i++;
                    # stay here, eating lines, until next NOT excluded item
                    push(@skipped,$line);
                    for (; $i < $lncnt; $i++) {
                        $line = $lines[$i];
                        chomp $line;
            			$lnnum = $i + 1;
                        if ($line =~ /^diff\s+.+/) {
            			#if ( $line =~ /^diff\s+(.+)\s+(.+)$/ ) {
                            $sline = $line;
                            $sline =~ s/^diff\s+//;
                            $fil1 = '';
                            $fil2 = '';
                            $cnt = get_diff_files($sline,\$fil1,\$fil2);
                            if (($cnt != 2)||(length($fil1)==0)) {
                               prtw("WARNING: get_diff_files:2: FAILED on [$sline] $cnt f1=[$fil1] f2=[$fil2]\n");
                            }
                            get_common_dst($fil1);
                            get_common_src($fil2);
                            #$res = is_excluded_file($line);
                            $res = is_excluded_file($fil1);
                            push(@excluded_files,[$fil1,$fil2,$res]) if ($res);
                            prt("[v5]$lnnum: $line [$fil1] ($res) 2\n") if (VERB5());
                            if (length($fil1) && !defined $dupes{$fil1}) {
                                ($name,$dir) = fileparse($fil1);
                                #                    0     1     2     3    4
                                push(@changed_files,[$name,$fil1,$fil2,$res,0]);
                                $dupes{$fil1} = 1;
                            }
                            last if ( !$res );
                            push(@skipped,$line);
                        } elsif ($line =~ /^Only\s+in\s+/) {
                            if ($KEEP_ONLY_IN) {
                                push(@only,$line); # -KO keep ALL 'Only in... lines
                            } elsif ($keep_only_in) {
                                #$res = is_excluded_file($line);
                                #push(@only,$line) if ($res <= $repo_dirs);   # EXCLUDE \.git\, \.svn\, etc... lines
                                push(@only,$line) if (!is_repo_dir($line));   # EXCLUDE \.git\, \.svn\, etc... lines
                            }
                            next;
                        } elsif ($line =~ /^\s*Binary\s+files\s+/) {
                            if ($keep_bin_files) {
                                $res = is_excluded_file($line);
                                push(@bins,$line) if ($res <= $repo_dirs);   # EXCLUDE \.git\, \.svn\, etc... lines
                            }
                            next;
                        } elsif ($line =~ /^\@\@\s+(.+)\s+\@\@\s*(.*)$/) {
                            $tmp = $1;
                            $extra = $2;    # git puts this after the numbers - is actually the next line!!!
                            $lcnts = $tmp;
                            $poff1 = $off1;
                            $plen1 = $len1;
                            $poff2 = $off2;
                            $plen2 = $len2;
                            get_counts($lcnts,\$off1,\$len1,\$off2,\$len2);
                            push(@excluded_new_files,$fil1) if (($off1 == 0)&&($len1 == 0));
                        } else {
                            push(@skipped,$line);
                        }
                    }
                    last if ($i == $lncnt);
                }
			} elsif ($line =~ /^Only\s+in\s+/) {
                if ($KEEP_ONLY_IN) {
                     push(@only,$line); # -KO keep ALL 'Only in... lines
                } elsif ($keep_only_in) {
                    #$res = is_excluded_file($line);
                    #push(@only,$line) if ($res <= $repo_dirs);   # EXCLUDE \.git\, \.svn\, etc... lines
                    push(@only,$line) if (!is_repo_dir($line));   # EXCLUDE \.git\, \.svn\, etc... lines
                }
                next;
			} elsif ($line =~ /^\s*Binary\s+files\s+/) {
                if ($keep_bin_files) {
                    $res = is_excluded_file($line);
                    push(@bins,$line) if ($res <= $repo_dirs);   # EXCLUDE \.git\, \.svn\, etc... lines
                }
                next;
            } elsif ($line =~ /^\@\@\s+(.+)\s+\@\@\s*(.*)$/) {
                $tmp = $1;
                $extra = $2;    # git puts this after the numbers - is actually the next line!!!
                $lcnts = $tmp;
                $poff1 = $off1;
                $plen1 = $len1;
                $poff2 = $off2;
                $plen2 = $len2;
                get_counts($lcnts,\$off1,\$len1,\$off2,\$len2);
                push(@new_files,$fil1) if (($off1 == 0)&&($len1 == 0));
            } elsif ($line =~ /^=================(=+)\s(\w+)/) {
                $proj = $2;
                if ($keep_only_in || $KEEP_ONLY_IN) {
                    push(@only,$line);
                }
            }
			push(@nlines,$line);
			$newlns++;
		}
        if ($keep_only_in || $KEEP_ONLY_IN) {
            $cnt = scalar @only;
            if ($cnt) {
                push(@nlines, "##################################################");
                push(@nlines, "### following $cnt lines added due to -ko option ###");
                push(@nlines, "##################################################");
                push(@nlines, sort(@only));
                push(@nlines, "##################################################");
            }
        }
        if ($keep_bin_files) {
            push(@nlines, @bins);
        }
        if ($add_skipped_lines) {
            $cnt = scalar @skipped;
            if ($cnt) {
                push(@nlines, "##################################################");
                push(@nlines, "### following $cnt lines added due to -a option ###");
                push(@nlines, "##################################################");
                push(@nlines, @skipped);
                push(@nlines, "##################################################");
            }
        }
	} else {
		prt( "ERROR: Unable to OPEN/READ file $fil ... check name, location...\n" );
	}
    return \@nlines;    # return the array reference
}

sub write_out_lines($$) {
    my ($rnls,$out) = @_;
    my $cnt = scalar @{$rnls};
    my ($txt);
    if ($cnt) {
        $txt = join("\n",@{$rnls});
        $txt .= "\n";
        write2file($txt,$out);
        prt("Written $cnt lines to [$out]...\n" );
    } else {
        $txt = "Appears no differences after cleaning\n";
        write2file($txt,$out);
        prt("Blank diffs written to $out\n");
    }
}

sub show_changed_files()
{
    my ($i,$name,$fil1,$fil2,$res,$cnt);
    my ($msg,$txt,$rescnt,$dst,$src,$ra,$rea,$cres,$added,$removed);
    my @newlist = ();
    my ($com_dst,$com_src,$len);
    my ($ncnt,$nxcnt,$tnew,$dfil,$inew,$outlen,$newcnt,$newxcnt);
#    my %dupes = ();
    $outlen = length($output_file);
    $newcnt = 0;
    $newxcnt = 0;
#    if (VERB1()) {
#        $rea = \@excluded_files;
#        $cnt = scalar @{$rea};
#        #prt("Excluded $cnt files from clean diff.\n") if (VERB9());
#        $rescnt = 0;
#        for ($i = 0; $i < $cnt; $i++) {
#            $fil1 = ${$rea}[$i][0];  # old - destination
#            $fil2 = ${$rea}[$i][1];  # new - source
#            $res  = ${$rea}[$i][2];  # reason
#            if (!defined $dupes{$fil1}) {
#                $dupes{$fil1} = 1;
#                $rescnt++;
#                #$cres = sprintf("%3d:%02d",$rescnt,$res);
#                #prt("$cres: [$fil1]\n") if (VERB9());
#            }
#        }
#    }

    $cnt = scalar @changed_files;
    $ncnt = scalar @new_files;
    $nxcnt = scalar @excluded_new_files;
    $tnew = $ncnt + $nxcnt;
    $msg = "Original diff contained $cnt changed ($tnew new) files...";
    $msg .= " use -v1 to view" if (!VERB1());
    prt("$msg\n");
    $ra = \@changed_files;
    # establish the NEW flag
    #                      0     1     2     3    4
    # push(@changed_files,[$name,$fil1,$fil2,$res,0]);
    for ($i = 0; $i < $cnt; $i++) {
        $fil1 = ${$ra}[$i][1];  # destination
        $res  = ${$ra}[$i][3];
        if ($res) {
            $inew = is_in_array($fil1,@excluded_new_files);
        } else {
            $inew = is_in_array($fil1,@new_files);
        }
        ${$ra}[$i][4] = $inew;
    }
    if ($cnt) {
        if ($show_copy_list) {
            $msg = '';
            $txt = '';
            $rescnt = 0;
            $dst = '';
            $src = '';
            $ra = \@changed_files;
            if (length($common_destination) && length($common_source)) {
                @newlist = ();
                $com_dst = $common_destination;
                $com_src = $common_source;
                $com_dst =~ s/\\/\\\\/g;
                $com_src =~ s/\\/\\\\/g;
                for ($i = 0; $i < $cnt; $i++) {
                    $name = ${$ra}[$i][0];
                    $fil1 = ${$ra}[$i][1];  # destination
                    $fil2 = ${$ra}[$i][2];  # source
                    $res  = ${$ra}[$i][3];
                    $inew = ${$ra}[$i][4];
                    $fil1 =~ s/$com_dst//;
                    $fil2 =~ s/$com_src//;
                    push(@newlist,[$name,$fil1,$fil2,$res,$inew]);
                }
                $src = "\%SRC\%";
                $dst = "\%DST\%";
                $ra = \@newlist;
                $txt .= "\@setlocal\n";
                $txt .= "\@set SRC=$common_source\n";
                $txt .= "\@set DST=$common_destination\n";
            }
            for ($i = 0; $i < $cnt; $i++) {
                $name = ${$ra}[$i][0];
                $fil1 = ${$ra}[$i][1];
                $fil2 = ${$ra}[$i][2];
                $res  = ${$ra}[$i][3];
                $inew = ${$ra}[$i][4];
                if ($res) {
                    $rescnt++;
                } else {
                    if ($inew) {
                        $msg = "copy $src$fil2 $dst$fil1";
                        prt("$msg\n") if (VERB9() || !$outlen);
                        $txt .= "$msg\n";
                    } else {
                        $newcnt++;  # well actually NOT NEW == OVERWRITE
                    }
                }
            }
            if ($newcnt) {
                $msg = "\@echo NOTE: Have $newcnt files which will OVERWRITE existing!";
                prt("$msg\n");
                $txt .= "$msg\n";
                $txt .= "\@echo The diff of these files should be reviewed BEFORE doing a COPY!\n";
                $txt .= "\@pause\n";
                for ($i = 0; $i < $cnt; $i++) {
                    $name = ${$ra}[$i][0];
                    $fil1 = ${$ra}[$i][1];
                    $fil2 = ${$ra}[$i][2];
                    $res  = ${$ra}[$i][3];
                    $inew = ${$ra}[$i][4];
                    if (!$res && !$inew) {
                       $msg = "copy $src$fil2 $dst$fil1";
                       prt("$msg\n") if (VERB9() || !$outlen);
                       $txt .= "$msg\n";
                    }
                }
            }
            if ($rescnt) {
                $msg = "\@echo NOTE: Have $rescnt files EXCLUDED from diff review file!";
                prt("$msg\n");
                $txt .= "$msg\n";
                $txt .= "\@echo The diff of these files should be reviewed BEFORE doing a COPY!\n";
                $txt .= "\@pause\n";
                for ($i = 0; $i < $cnt; $i++) {
                    $name = ${$ra}[$i][0];
                    $fil1 = ${$ra}[$i][1];
                    $fil2 = ${$ra}[$i][2];
                    $res  = ${$ra}[$i][3];
                    if ($res) {
                        $msg = "\@echo copy $src$fil2 $dst$fil1";
                        prt("$msg\n") if (VERB9() || !$outlen);
                        $txt .= "$msg\n";
                    }
                }
            }
            if (length($output_file)) {
                write2file($txt,$output_file);
                prt("Copy list written to $output_file\n");
            }
        } elsif (VERB1()) {
            $rescnt = 0;
            $added = 0;
            $removed = 0;
            $len = length($common_destination);
            for ($i = 0; $i < $cnt; $i++) {
                #$name = $changed_files[$i][0];
                #$fil1 = $changed_files[$i][1];
                #$fil2 = $changed_files[$i][2];
                $res  = $changed_files[$i][3];
                if ($res) {
                    $removed++;
                } else {
                    $added++;
                }
            }
            $msg = "Total $removed files removed from cleaned diff...";
            $msg .= " use -v5 to view." if (!VERB5());
            prt("$msg\n");
            for ($i = 0; $i < $cnt; $i++) {
                $name = $changed_files[$i][0];
                $fil1 = $changed_files[$i][1];
                $fil2 = $changed_files[$i][2];
                $res  = $changed_files[$i][3];
                $inew = $changed_files[$i][4];
                if ($res) {
                    $rescnt++;
                    $cres = sprintf("%3d",$rescnt);
                    $dfil = substr($fil1,$len);
                    $msg = "$cres:REMOVED: [$dfil]";
                    #prt(" $fil1 ($res)\n");
                    $msg .= " NEW" if ($inew);
                    prt("$msg\n") if (VERB5());
                }
            }
            $msg = "Total $added files included in cleaned diff...";
            $msg .= " use -v2 to view." if (!VERB2());
            prt("$msg\n");
            $added = 0;
            for ($i = 0; $i < $cnt; $i++) {
                $name = $changed_files[$i][0];
                $fil1 = $changed_files[$i][1];
                $fil2 = $changed_files[$i][2];
                $res  = $changed_files[$i][3];
                $inew = $changed_files[$i][4];
                if (!$res) {
                    $rescnt++;
                    $added++;
                    $cres = sprintf("%3d:%3d",$added,$rescnt);
                    $dfil = substr($fil1,$len);
                    $msg = "$cres: [$dfil]";
                    #prt(" $fil1 ($res)\n");
                    $msg .= " NEW" if ($inew);
                    prt("$msg\n") if (VERB2());;
                }
            }
        }
    } else {
        if (length($output_file)) {
            $txt = "Appears no list after cleaning\n";
            write2file($txt,$output_file);
            prt("Blank list written to $output_file\n");
        }
    }
    if (VERB5()) {
        prt("Common dst [$common_destination]\n");
        prt("Common src [$common_source]\n");
    }
}

# =======================================
# MAIN

parse_args(@ARGV);
my $ref_lines = process_file($in_file);
write_out_lines($ref_lines,$out_file);
show_changed_files();
pgm_exit(0,"");

# =======================================

sub load_input_file($$) {
    my ($arg,$file) = @_;
    if (open INF, "<$file") {
        my @lines = <INF>;
        close INF;
        my @carr = ();
        my ($line,@arr,$tmp,$i);
        my $lncnt = scalar @lines;
        for ($i = 0; $i < $lncnt; $i++) {
            $line = $lines[$i];
            $line = trim_all($line);
            next if (length($line) == 0);
            next if ($line =~ /^#/);
            while (($line =~ /\\$/)&&(($i+1) < $lncnt)) {
                $i++;
                $line =~ s/\\$//;
                $line .= trim_all($lines[$i]);
            }
            #@arr = split(/\s/,$line);
            @arr = space_split($line);
            foreach $tmp (@arr) {
                $tmp = local_strip_both_quotes($tmp);
                push(@carr,$tmp);
            }
        }
        $in_input_file++;
        parse_args(@carr);
        $in_input_file--;
    } else {
        pgm_exit(1,"ERROR: Unable to 'open' file [$file]!\n")
    }
}

sub need_arg {
    my ($arg,@av) = @_;
    pgm_exit(1,"ERROR: Arg [$arg] must be followed by a 2nd argument!\n") if (!@av);
}

sub local_strip_both_quotes($) {
    my $txt = shift;
    if ($txt =~ /^'(.+)'$/) {
        return $1;
    }
    if ($txt =~ /^"(.+)"$/) {
        return $1;
    }
    return '' if ($txt eq '""');
    return '' if ($txt eq "''");
    #prt("Stripping [$txt] FAILED\n");
    return $txt;
}

sub parse_args {
	my (@av) = @_;
	my ($arg,$ch,$sarg,@arr,$tmp);
	while (@av) {
		$arg = $av[0];
		$ch = substr($arg,0,1);
		if ($arg =~ /\?/) {
			give_help();
        	pgm_exit(0,"Help exit\n");
		} elsif (($ch eq '-')||($ch eq '/')) {
            $sarg = substr($arg,1);
            $sarg = substr($sarg,1) while ($sarg =~ /^-/);
			#$val = substr($arg,1);
            if ($sarg =~ /^a/) {
                $add_skipped_lines = 1;
                prt("Adding skipped lines to end of review file\n") if (VERB2());
            } elsif ($sarg =~ /^h/) {
                give_help();
            	pgm_exit(0,"Help exit\n");
            } elsif ($sarg =~ /^c/) {
                $sarg = substr($sarg,1);
                $show_copy_list = 1;
                if (length($sarg)) {
                    $output_file = $sarg;
                    prt("Output copy list to $output_file.\n") if (VERB2());
                } else {
                    prt("Output a copy list.\n") if (VERB2());
                }
#            } elsif ($sarg =~ /^i/) {
#                if ($sarg =~ /^in:(.+)$/) {
#    				$in_file = substr($sarg,3);
#                } else {
#                    need_arg(@av);
#                    shift @av;
#                    $sarg = $av[0];
#				    $in_file = $sarg;
#                }
#				prt( "Set input file to [$in_file] ...\n" ) if (VERB2());
            } elsif ($sarg =~ /^o/) {
    			if ($sarg =~ /^out:(.+)$/) {
    				$out_file = substr($sarg,4);
                } else {
                    need_arg(@av);
                    shift @av;
                    $sarg = $av[0];
			    	$out_file = $sarg;
                }
				prt( "Set output file to [$out_file] ...\n" ) if (VERB2());
            } elsif ($sarg eq 'ko') {
                $keep_only_in = 1;
				prt( "Set to keep 'Only in ' lines, except in repo folder...\n" ) if (VERB2());
            } elsif ($sarg eq 'KO') {
                $KEEP_ONLY_IN = 1;
				prt( "Set to keep ALL 'Only in ' lines...\n" ) if (VERB2());
            } elsif ($sarg eq 'kb') {
                $keep_bin_files = 1;
                prt("Set to keep 'Binary files ' lines...\n") if (VERB2());
            } elsif ($sarg eq 'll') {
                $load_log = 1;
				prt( "Set to load log at end...\n" ) if (VERB2());
            } elsif ($sarg eq 'm') {
                $include_msvc = 1;
				prt( "Set to include MSVC files.\n" ) if (VERB2());
            } elsif ($sarg =~ /^x/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                if ($sarg eq '_cmake_') {
                    push(@user_xcludes, @cmake_xcludes);
                    prt("Added [$sarg] excludes.\n") if (VERB2());
                } else {
                    @arr = split(";",$sarg);
                    foreach $sarg (@arr) {
                        push(@user_xcludes,$sarg);
                        prt("Added [$sarg] to user file excludes.\n") if (VERB2());
                    }
                }
            } elsif ($sarg =~ /^d/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                push(@xclude_dirs,$sarg);
                prt("Added [$sarg] to user directory excludes.\n") if (VERB2());
            } elsif ($sarg =~ /^v/) {
                if ($sarg =~ /^v(\d+)$/) {
                    $verbosity = $1;
                } else {
                    while ($sarg =~ /^v/) {
                        $verbosity++;
                        $sarg = substr($sarg,1);
                    }
                }
                prt("Set verbosity to [$verbosity]\n") if (VERB2());
            } elsif ($sarg =~ /^\@/) {
                $sarg = substr($sarg,1);
                prt("Loading from input file [$sarg]\n") if (VERB2());
                load_input_file($arg,$sarg);
            } elsif ($sarg =~ /^i/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                prt("Loading from input file [$sarg]\n") if (VERB2());
                load_input_file($arg,$sarg);
            } elsif ($sarg =~ /^o/) {
                need_arg(@av);
                shift @av;
                $sarg = $av[0];
                $out_file = $sarg;
                prt("Set the output file to [$out_file].\n") if (VERB2());
 #           } elsif ($sarg =~ /^u/) {
 #               $unix_pat_sep = 1;
 #               prt("Set the unix path separator.\n");
            } else {
				prt( "ERROR: Unknown argument [$arg]!\n" );
				give_help();
            	pgm_exit(1,"Error exit\n");
			}
		} else {
            # assume a bare argument in the INPUT file
    		$in_file = $arg;
			prt( "Set input file to [$in_file] ...\n" ) if (VERB2());
		}
        shift @av;
	}

    if ($in_input_file == 0) {
        if (length($in_file) == 0) {
            prt( "ERROR: NO input file found in command! Aborting...\n" );
            pgm_exit(1,"No file exit\n");
        } elsif ( ! -f $in_file) {
            prt( "ERROR: Unable to locate [$in_file] file! Aborting...\n" );
            pgm_exit(1,"No file exit\n");
        }
    }
}

# # 23/03/2012 - Add -u to convert paths to unix path speparator
sub give_help {
	prt( "Brief HELP for $pgmname script ... version $VERS\n" );
	prt( "$pgmname [options] -\@input input_diff\n" );
    prt( "options:\n" );
    prt( " -a         = Add skipped lines to END of diff review file.\n");
    prt( " -c[output] = Write a copy changed list. To [output], if given.\n");
    prt( " -i <input> = Load commands from input file, space, line sepearted items.\n");
    prt( " -ko        = Keep 'Only in ...' lines, except in .git folder, and add to end.\n" );
    prt( " -KO        = Keep ALL 'Only in ...' lines, and add to end.\n" );
    prt( " -ll        = Load log at end.\n" );
    prt( " -m         = Included all MSVC files, normally excluded.\n");
    prt( " -v[n]      = Bump or set verbosity. Def=$verbosity.\n");
    prt( " -x <file>  = Exclude this file from the cleaned diff.\n");
    prt( " This can be a ';' separated list, and can include wild cards, like *.html\n");
    prt( " -x _cmake_ = Special exclude of CMake file changes.\n");
    prt( " -d <dir>   = Exclude files with this folder from the cleaned diff.\n");
    prt( " -o <file>  = Set the OUTPUT cleaned diff file.\n");
#    prt( " -u         = Use unix path separator.\n");
	prt( "Defaults: in:[$in_file], out:[$out_file]\n" );
    prt( "Will exit if error found in command!\n" );
}


# eof - cleandiff02.pl