#!/usr/bin/perl
use locale;
use utf8;
#use encoding "utf-8";
@snt_array;

open(FILE, "tagset.postag") || die;
binmode(FILE,":utf8");
while($line=<FILE>){
    $line=~s/\n//;
    $postag{$line}++;
}
close FILE;

       my $filename=$ARGV[0];
    
    
	$rname="generatedconllfromxml.conll";
    
    
    $num=0;
    $ipname="newdata.conll.temp";
    
    
    open(newFILE, ">$ipname") || die;
    binmode(newFILE,":utf8");

    open(reindexFILE, ">$rname") || die;
    binmode(reindexFILE,":utf8");


    local *FILE;
    open(FILE, $filename) || die;
    binmode(FILE,":utf8");
    $list[$ii]=~/([\w]+\.xml)/;
    close newFILE;
    open(newFILE, ">>$ipname") || die;
    binmode(newFILE,":utf8");
    $line =<FILE>;
    while ($line ){	
	$ix=0;	
	if ($line=~ m/<\/[S]/){
	    $line=~ s/<\/[S]//;
	}
	CumleVeriyapisi();	
	shift @snt_array;
	if (@snt_array==0){ #eger bossa o zaman cik
		next;
	}
	
	$slen=@snt_array;
	#FOR REINDEXING
	$currsumig=1;
	foreach $item (@snt_array){
	    $newindex[$item->{"ix"}]=$currsumig;
	    $currsumig+=$item->{"igcount"};
	}
	$nind=0;
	undef @mycolumns;
	$toreindex=0;
	undef @etolind;
	
	push @mycolumns, "";
	foreach $item (@snt_array){
	    undef @columns;	$columns[8]="_";    $columns[9]="_\n";
	    if($item->{"igcount"}>1){
		@myig=split(/\+/,$item->{"maltpos"});		    
		for($igt=1;$igt<=$item->{"igcount"};$igt++){
		    $nind++;
		    undef @columns;	$columns[8]="_";    $columns[9]="_\n";
		    $columns[0]=$nind;
		    $columns[1]="_";
		    if($igt==$item->{"igcount"}){
			$columns[1]=$item->{"word"};
		    }
		    $columns[2]="_";
		    if($igt==1){
			$columns[2]=$item->{"stem"};
		    }
		    $columns[3]=$item->{$igt};
		    $columns[4]=$item->{$igt."npos"};
		    $columns[5]=$item->{$igt."feats"};
		    if($igt==$item->{"igcount"}){
			if($item->{"next"}==NULL){
			    if($item->{"finalpos"} eq "Punc" && $item->{"ix"}!=$slen){
				#$columns[6]=$newindex[$item->{"ix"}+1];
				$columns[6]=0;
				#$columns[7]="notconnected";
				$columns[7]="ROOT";
			    }
			    else{
				$columns[6]=0;
				$columns[7]="ROOT";
			    }
			}
			else{
			    $columns[6]=$newindex[$item->{"next"}]+$item->{"nextig"}-1;
			    $columns[7]=$item->{"REL"};
			}
		    }
		    else{
			$columns[6]=$newindex[$item->{"ix"}]+$igt;
			$columns[7]="DERIV";
		    }
		    converttags();
		    $cpostag{$columns[3]}=1;$npostag{$columns[4]}=1; $rel{$columns[7]}=1;		    
		    $lineor=$columns[0];for($i=1;$i<10;$i++){$lineor.="\t".$columns[$i];}
		    print newFILE $lineor;
		    if($columns[7] eq "MWE" && $columns[6]==$columns[0]+1){
			$toreindex++;
			push @etolind,$nind;
		    }
		    push @mycolumns, [@columns];    
		}	
		
	    }
	    else{
		$nind++;
		$columns[0]=$nind;
		$columns[1]=$item->{"word"};
		$columns[2]=$item->{"stem"};
		$columns[3]=$item->{"1"};
		$columns[4]=$item->{"1npos"};
		$columns[5]=$item->{"1feats"};
		if($item->{"next"}==NULL){
		    if($item->{"finalpos"} eq "Punc" && $item->{"ix"}!=$slen){
			#$columns[6]=$newindex[$item->{"ix"}+1];
			$columns[6]=0;
			#$columns[7]="notconnected";
			$columns[7]="ROOT";
		    }
		    else{
			$columns[6]=0;
			$columns[7]="ROOT";
		    }
		}
		else{
		    $columns[6]=$newindex[$item->{"next"}]+$item->{"nextig"}-1;
		    $columns[7]=$item->{"REL"};
		}
		converttags();
		$cpostag{$columns[3]}=1;$npostag{$columns[4]}=1; $rel{$columns[7]}=1;		
		$lineor=$columns[0];for($i=1;$i<10;$i++){$lineor.="\t".$columns[$i];}
		if($columns[7] eq "MWE" && $columns[6]==$columns[0]+1){
		    $toreindex++;
		    push @etolind,$nind;
		}
		push @mycolumns, [@columns];    		
		print newFILE $lineor;        		
	    }
	    
	}
	
	
	print newFILE "\n";
	
	$k=@snt_array;
	while ($k!=0){
	    $k--;
	    pop @snt_array;
	}
	reindex_sentence();
	
    }
    close newFILE;




open(newFILE, ">tagset.cpos") || die;
binmode(newFILE,":utf8");
    foreach $name (sort keys  %cpostag){
	if($name ne ""){
	print newFILE "$name\n";
	}
    }
close newFILE;
open(newFILE, ">tagset.pos") || die;
binmode(newFILE,":utf8");
    foreach $name (sort keys  %npostag){
	if($name ne ""){
	print newFILE "$name\n";
	}
    }
close newFILE;
open(newFILE, ">tagset.dep") || die;
binmode(newFILE,":utf8");
    print newFILE "ROOT\n";
    #print newFILE "notconnected\n";
    print newFILE "DERIV\n";
    
    foreach $name (sort keys  %rel){
	if($name ne ""){
	print newFILE "$name\n";
	}
	
    }
close newFILE;
sub CumleVeriyapisi{
    while ($line && $line!~ m/<\/[S]/){   
	    	    
	    if($line=~m/IG=\'\[(.*)\]\' REL/g){
		$ix++;
		$pos=$1;	
		
		$snt_array[$ix]->{"pos"}=$pos;
		$deneme=$pos;
		#$deneme=~m/\"([\w\,\.\_\'\-\?\!]+\;\:)\+/;
		if($deneme=~m/\"([\w\,\.\_\'\"\(\)\-\?\!\;\&\:]+)\+/){
		    $snt_array[$ix]->{"stem"}=$1;
		    if($snt_array[$ix]->{"stem"} eq "&quot;"){
			$snt_array[$ix]->{"stem"}="\"";
		    }
		}
		if($deneme=~/CASA/){
		    $ok;
		}
		#$snt_array[$ix]->{"stem"}=$1;
		$snt_array[$ix]->{"igcount"}=0;
		$deneme=$pos;
		undef $maltpos;

		#while ($deneme=~m/\(([\w\,\.\_\'\-\+\"\?\!\:\;]+||\(||\))\)/g){
		while ($deneme=~m/\(([\w\,\.\_\'\-\+\"\?\!\:\;\&\)]+)\)/g){
		    $snt_array[$ix]->{"igcount"}++;
		    
		    $search= $1;
		    $im="ig".$snt_array[$ix]->{"igcount"};
		    $snt_array[$ix]->{$im}=$search;

		    if($deneme=~m/\)\+Punc\"\)/){
			$search="1,\"+Punc\"";
		    }
		    @myig=split(/,/,$search);		    
		
		    $temp=$search;
		    @parts=split(/\+/,$search);	    
		    
		    undef $dpos;
		    if ($myig[0]==1){
			$search=~m/[\w\"\,\.\)\'\:\?\-\;\&\(\!]+\+([A-Z]\w+)/;
			$search=$1;
			undef $parts[0];
			undef $parts[1];
		    }
		    else{
			$search=~m/([A-Z]\w+)/;
			$search=$1;
			undef $parts[0];
		    }
		    $dpos=$search;
		    foreach $part (@parts){
			$part=~s/\"//;
			if(exists($postag{$part})){
			    if($part eq "FutPart" ||$part eq "PastPart"||$part eq "PresPart"||$part eq "Inf"){
				if($search eq "Noun"){
				    $part="N".$part;
				}
				else{
				    $part="A".$part;
				}
			    }
			    $dpos=$part;
			    undef $part;			    
			}
		    }
		    undef $feats;
		    foreach $part (@parts){
			if(defined $part){			    
			    $feats.=$part."|";
			}
		    }
		    $feats=~s/\|$//;
		    if(!defined $feats){
			$feats="_";
		    }
		    $snt_array[$ix]->{$snt_array[$ix]->{"igcount"}}=$search;
		    $snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."npos"}=$dpos;
		    $snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."feats"}=$feats;
		    
		    if(!defined $maltpos){
			$maltpos=$search;
		    }
		    else{
			$maltpos.="+".$search;
			#$maltpos=$search;
		    }
		    if($temp=~m/\+(P1sg)/ || $temp=~m/\+(P2sg)/ || $temp=~m/\+(P3sg)/|| $temp=~m/\+(P1pl)/ || $temp=~m/\+(P2pl)/ ||$temp=~m/\+(P3pl)/ ||$temp=~m/\+(Pnon)/){			
                    $snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."poss"}=$1;
			if($search eq "Noun"||$search eq "Pron"){
                        $temp=~m/\+([A-Z]\w+)\+([A-Z]\w+)\+([A-Z]\w+)\"/;
			$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."A"}=$1;			
			$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."poss"}=$2;			
                        $snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."case"}=$3;
			$maltpos.="^".$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."case"}."^".$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."poss"};
			
			#$maltpos=~s/Noun$//;
			#$maltpos=~s/Pron$//;
			#$maltpos.=$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."case"}."^".$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."poss"};
			
			#$maltpos.="^".$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."case"};                        
			}
			##newly included
			#elsif($search eq "Adj"){			    
			#    $maltpos.="^".$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."poss"};
			#}
			
		    }
		    if($search eq "Verb" && ($temp=~m/\+(A1sg)/ || $temp=~m/\+(A2sg)/ || $temp=~m/\+(A3sg)/|| $temp=~m/\+(A1pl)/ || $temp=~m/\+(A2pl)/ ||$temp=~m/\+(A3pl)/ )){			
			$snt_array[$ix]->{$snt_array[$ix]->{"igcount"}."A"}=$1;			
		    }
		}
		
		if($temp=~m/\+(P1sg)/ || $temp=~m/\+(P2sg)/ || $temp=~m/\+(P3sg)/|| $temp=~m/\+(P1pl)/ || $temp=~m/\+(P2pl)/ ||$temp=~m/\+(P3pl)/ ||$temp=~m/\+(Pnon)/){			
		$snt_array[$ix]->{"poss"}=$1;		    
		    if ($search eq "Noun"||$search eq "Pron"){
		    $temp=~m/\+([A-Z]\w+)\+([A-Z]\w+)\+([A-Z]\w+)\"/;
		    $snt_array[$ix]->{"A"}=$1;	
		    $snt_array[$ix]->{"poss"}=$2;		    
		    $snt_array[$ix]->{"case"}=$3;
		    }
		}
		$snt_array[$ix]->{"finalpos"}=$search;
		#if (!exists($gshash{$snt_array[$ix]->{"finalpos"}})) {
		#    print ("hata");
		#}	
		
		@arr=split(/\" LEM/,$line);    			
		@arr=split(/W IX=\"/,$arr[0]);
		$curr=$arr[1];
		$snt_array[$ix]->{"ix"}=$curr;
		@arr=split(/REL=\"\[/,$line);    			
		@arr=split(/\,/,$arr[1]);
		$next=$arr[0];
		$snt_array[$ix]->{"next"}=$next;
		if($next!=NULL){
		    $snt_array[$ix]->{"nextig"}=$arr[1];
		}
		@arr=split(/REL=\"\[/,$line);
		@arr=split(/>/,$arr[1]);
		@arr=split(/\,/,$arr[0]);
		@arr=split(/\)/,$arr[2]);
		@arr=split(/\(/,$arr[0]);
		$snt_array[$ix]->{"REL"}=$arr[1];
		$line=~m/> ([\w\"\,\.\)\'\:\?\-\;\!]+) <\/[W]>/;
		$try=$1;
		if($try=~m/\(/){
		    $try="(";
		}
#    		if($maltpos=~/Punc/||$maltpos eq ""){
#		    $maltpos=$try."+Punc";
#		}
		
		$snt_array[$ix]->{"maltpos"}=$maltpos;
		$snt_array[$ix]->{"word"}=$try;
		$snt_array[$ix]->{"connected"}=0;
	    }
	    $line =<FILE>;
	}#while d�ng�s�n�n sonu bir c�mleyi veri yap?s?na aktar?r.

}


sub converttags{
    if($columns[5]=~/^Inf1/){	
	$columns[5]=~s/Inf1/Inf/;
    }
    elsif($columns[5]=~/^Inf2/){	
	$columns[5]=~s/Inf2/Inf/;
    }
    elsif($columns[5]=~/^Inf3/){	
	$columns[5]=~s/Inf3/Inf/;
    }
    if($columns[5]=~/(FutPart)/ || $columns[5]=~/(PastPart)/ || $columns[5]=~/(PresPart)/  || $columns[5]=~/(Inf)/  ){
	$pt=$1;
	if($columns[4] eq "Adj"){
	    $columns[4]="A".$pt;
	}
	elsif($columns[4] eq "Noun"){
	    $columns[4]="N".$pt;
	}
	if($columns[5]=~/$pt\|/){
	    $columns[5]=~s/$pt\|//;
	
	}
	elsif($columns[5] eq $pt){
	    $columns[5]="_";
	}	
    }
    elsif($columns[3] eq "Pron" && $columns[4] eq "Ques"){
	$columns[4] = "QuesP";
    }
    elsif($columns[3] eq "Pron" && ($columns[5]=~/(Demons)\|/ || $columns[5]=~/(Pers)\|/ || $columns[5]=~/(Reflex)\|/) ){
	$pt=$1;
	$columns[5]=~s/$pt\|//;
	$columns[4] = $pt."P";
    }
    if($columns[4] eq "Pron" && $columns[5]=~/Quant\|/){
	$columns[5]=~s/Quant\|//;
    }
}
sub reindex_sentence{
$csize=$#mycolumns;
 if($toreindex==0){
    yazdir();
    return;
 }
 
 while($toreindex>0){
    $ri=pop @etolind;    
    $columns=splice(@mycolumns, $ri, 1);    
    $csize--;    
    for($k=$ri;$k<=$csize;$k++){
	$mycolumns[$k][0]--;
    }
    $mycolumns[$ri][2]=$columns->[1]."_".$mycolumns[$ri][2];
    $rii=$ri;
    while($mycolumns[$rii][1] eq "_"){
	$rii++;
    }
    $mycolumns[$rii][1]=$columns->[1]."_".$mycolumns[$rii][1];
    for($k=1;$k<$csize;$k++){
	if($mycolumns[$k][6]>$ri){
	    $mycolumns[$k][6]--;
	}	
    }
  $toreindex--;  
 }
  yazdir();
}
sub yazdir{
    for($k=1;$k<=$csize;$k++){
	$lineor=$mycolumns[$k][0];for($i=1;$i<10;$i++){$lineor.="\t".$mycolumns[$k][$i];}
	print reindexFILE   $lineor;
    }
    print reindexFILE   "\n";
    
    print $sntnumbers++."\n";
}