%==============================================================================
%    evmodules.m
%    Copyright (C) 2007  Amoolya H. Singh, singh@embl.de
%
%    This program is free software: you can redistribute it and/or modify
%    it under the terms of the GNU General Public License as published by
%    the Free Software Foundation, either version 3 of the License, or
%    (at your option) any later version.
%
%    This program is distributed in the hope that it will be useful,
%    but WITHOUT ANY WARRANTY; without even the implied warranty of
%    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%    GNU General Public License for more details.
%
%    You should have received a copy of the GNU General Public License
%    along with this program.  If not, see <http://www.gnu.org/licenses/>.
%===============================================================================
function evmodules(pathway)
%------------------------------------------------------------------------------
%
% function evmodules(pathway) calculates evolutionary gene modules
% from phylogenetic and phenotypic data
%
% Input: 
%   pathway, is 'spo' 'com' or 'che'
%
%------------------------------------------------------------------------------

% -- import data
path=['../',pathway,'/']
file=[path,'sorted-tbl.txt']		% phylo profile
importdata(file);
M = ans.data;
ngenes = size(M,1);
G = ans.textdata;

[species_names] = importdata('../db/vimss-species.csv');
[taxids] = importdata('../db/taxids.txt');
%todelete=importdata('../db/duplicate-vimss-linenums.txt');
todelete = find_dup_genomes(0.9);

[c,ia,ib] = intersect(taxids,todelete);
species_names(ia)
species_names(ia) = [];
taxids(ia) = [];
M(:,ia) = [];

nspecies = size(M,2);


file=[path,'alpha-fn.txt']		% eng ontol
importdata(file);
EO = ans.data;
P = EO;

file=[path,'alpha-tigrfn.txt']		% tigr categories
importdata(file);
TGR = ans.data;
TGR1 = TGR(:,1); TGR2 = TGR(:,2);

file=[path,'alpha-cogclass.txt']	% COG fnal class
COG = importdata(file);

file=[path,'alpha-cog.txt']		% COG numbers
COGNUM = importdata(file);

file=[path,'alpha-medians.txt']		% median % ID
importdata(file);
MED = ans.data';

file=[path,'alpha-dnds.txt']		% dN/dS
importdata(file);
DN = ans.data';

% phylogenetic controls
importdata('../db/BSUB-tbl.txt');
SUB = ans.data;
SUB(:,64) = [];			        % duplicate column 
SUB(:,ia) = [];
Gsub = ans.textdata;

if (p=='com')
  importdata('../db/HIN-tbl.txt');
  HIN = ans.data;
  Ghin = ans.textdata;
end

% maximum possible distance (all 1s vs all 0s) used for normalizing
maxdist = pdist([ones(1,nspecies);zeros(1,nspecies)],'euclidean');


%------------------------------------------------------------------------------
% -- cluster the phylo profile matrix along genes

D = eudist(M); DD = squareform(D);
Z = linkage(D,'ward');

%------------------------------------------------------------------------------
% -- calculate optimal number of clusters from silhouette analysis
% first, calculate number of gene clusters
[SM,SIL] = silh(M,Z);
[c,i] = max(SM);
ngeneclusts = i+1;

% make dendrogram for public consumption, with all leaves
figure
[H,T,perm] = dendrogram(Z,0,'colorthreshold',5.8, ...
			'orientation','left','labels',G);
set(gca,'XGrid','on');
set(gca,'TickDir','out');
orient tall
%print('-depsc', '-cmyk','genes.eps');

%------------------------------------------------------------------------------
% -- get gene tree params
figure
[H,T,perm] = dendrogram(Z,ngeneclusts,'colorthreshold','default', ...
			'orientation','left');
set(gca,'XGrid','on');
set(gca,'TickDir','out');
[Hall,Tall,permall] = dendrogram(Z,0,'colorthreshold','default', ...
				 'orientation','left');

%------------------------------------------------------------------------------
% -- cluster the phylo profiel matrix along species
D2 = eudist(M');
Z2 = linkage(D2,'ward');

% calculate number of species clusters
[SM2,SIL2] = silh(M',Z2);
[c2,i] = max(SM2);
nspeciesclusts = i+1;

figure
[H2,T2,perm2] = dendrogram(Z2,nspeciesclusts,'colorthreshold','default', ...
			   'orientation','left');

[H2all,T2all,perm2all] = dendrogram(Z2,0,'colorthreshold',5.85, ...
			   'orientation','left');
set(gca,'YTickLabel',species_names(perm2all),'FontSize',4);
set(gca,'XGrid','on');
set(gca,'TickDir','out');
orient tall
%print('-depsc', '-cmyk','species.eps');

F4 = fopen([path,'/clustered-species-nodups.txt'],'w');
for i=1:nspecies
  fprintf(F4,'%d\t%s\n',taxids(perm2all(i)),...
	  char(species_names(perm2all(i))));
end

%------------------------------------------------------------------------------
% -- construct clustered files, matrices

% print clustered file in order
% C = M(perm,perm2);
% figure,checkerboard(C);


F2 = fopen([path,'/gene-clusters-nodups.txt'],'w');
F3 = fopen([path,'/species-clusters-nodups.txt'],'w');
F = fopen([path,'clustered-nodups.txt'],'w');

for j=1:length(perm2)
  ind = find(T2==perm2(j));
  fprintf(F3,'\nCluster %d\n',perm2(j));
  for k=1:length(ind)
    fprintf(F3,'%s\n',char(species_names(ind(k))));;
  end
end

% with species header line
fprintf(F,'NAME,');
for i=1:nspecies
  fprintf(F,'%s,',char(species_names(perm2all(i))));
end
fprintf(F,'\n');

% print clusters

Mclustered = M(permall, perm2all);

for i=1:ngenes
  ii = permall(i);
  fprintf(F,'%s',char(G(ii)));
  for j=1:nspecies
    jj = perm2all(j);
    fprintf(F,',%d',Mclustered(i,j));
  end
  fprintf(F,'\n');
end

Adj = zeros(ngenes);			% adjacency matrix
CM = zeros(ngenes);			% clustered matrix
permuteindex = [];
for i=1:length(perm)
  fprintf(F2, '\nCluster %d\n',perm(i));
  ind = find(T==perm(i));
  permuteindex = [permuteindex; ind];
  Adj(ind,ind) = 1;
  for j=1:length(ind)
    a = ind(j);
    fprintf(F2,'%s\n',char(G(a)));
  end
end
fclose(F);
fclose(F2);
fclose(F3);
fclose(F4);

for i=1:ngenes, Adj(i,i) = 0; end	% remove self loops
% clustered matrices 
CM = DD(permall,permall);
AM = Adj(permall,permall);
% weighted adjacency matrix
WA = AM.*CM;

%------------------------------------------------------------------------------
% -- input phenotype data
fid = fopen('../db/phenotypes-updated.csv');
%C =textscan(fid,'%s%s%s%s%s%s%s%s%s%s%s%s%*[^\n]','delimiter',';');
C = textscan(fid,'%s%s%s%s%s%s%s%s%s%s%s%s%s%*[^\n]','delimiter',';');
fclose(fid);

che_species = C{9};
com_species = C{10};
spo_species = C{11};
anipath_species = C{6};
plantpath_species = C{7};
anaer_species = C{8};
extremo_species = C{12};
grampos_species = C{13};
    
che_phenotypes = fill_phenotypes(che_species);
spo_phenotypes = fill_phenotypes(spo_species);
com_phenotypes = fill_phenotypes(com_species);
ani_phenotypes = fill_phenotypes(anipath_species);
pla_phenotypes = fill_phenotypes(plantpath_species);
ana_phenotypes = fill_phenotypes(anaer_species);
ext_phenotypes = fill_phenotypes(extremo_species);
gmp_phenotypes = fill_phenotypes(grampos_species);

PH = [che_phenotypes;spo_phenotypes;com_phenotypes;ani_phenotypes; ...
	pla_phenotypes;ana_phenotypes;ext_phenotypes;gmp_phenotypes];

pheno_names = {'Motile';'Spore-former';'Competent';...
	       'Animal Pathogen';'Plant Pathogen'; 'Anaerobe';...
	       'Extremophile';'Gram Positive'};

[unique_phenos,pheno_counts,phes] = unique(PH','rows');
numuniq = length(pheno_counts);
pheno_strings = cellstr(num2str(unique_phenos));

% cluster phenotype matrix, just for fun
% D3 = pdist(PH,'euclidean');
% Z3 = linkage(D3,'ward');
% [H3,T3,perm3] = dendrogram(Z3,0,'colorthreshold','default',...
% 			   'orientation','left');
% set(gca,'YTickLabel',pheno_names(perm3));
% set(gca,'XGrid','on');
% set(gca,'TickDir','out');
% orient tall
% print('-depsc', '-cmyk','phenos.eps');

% score which species have which modules
species_with_modules = zeros(ngeneclusts,nspecies);
S12 = [];
for i=1:ngeneclusts
  ind = find(T1==i);
  spe = find(sum(M(ind,:))>0.5*length(ind));
  species_with_modules(i,spe) = 1;
  freqs = histc(phes(spe),1:numuniq);
  S12(:,i) = freqs/sum(freqs);
end

% write to file
F = fopen('species_with_modules.txt','w');
fprintf(F,'COLOR,#0000ff,#00ff00,,,#ff0000,,#ff6666,#00ffff\n');
for i=1:nspecies
  %fprintf(F,'%s,%d,%d,%d,%d,%d,%d,%d,%d\n',...
  fprintf(F,'%s,%d,%d,%d,%d\n',...
	  char(species_names(i)),species_with_modules(:,i));
end
fclose(F);

% calculate phenotypic composition of each species cluster
% spec_S1 = clust_composition(T2,perm2,phes,pheno_strings);

% calculate phenotypic composition of each gene module
descr = pheno_strings;
ind = find(sum(S12')==0);
pheno_strings(ind) = [];
S(ind,:) = [];
ncategories = length(pheno_strings);
figure,bar(S','stacked');
axis tight
legend(pheno_strings);
print('-depsc', '-cmyk','pheno-to-module.eps');


% evaluate the next line in shell
% for i in 1 2 5 7 8; do echo $i; ii=`expr $i + 1`; echo $ii; cut -d',' -f1,$ii species_with_modules.txt > species_mod$i.txt; done  

%------------------------------------------------------------------------------
% -- calculate cross-phenotype sensitivity & specificity (that is, spo
% genes should agree best with spo phenotype, com with com,
% etc). uncomment the line for whichever network is being analysed


% che
% [chesens,chespec,cheerr] = sens_spec(che_phenotypes, M, pathway,'mot', T,P,G);

% % spo
[sposens,spospec,spoerr] = sens_spec(spo_phenotypes, M, pathway,'spo', T,P,G);

% % com
% [comsens,comspec,comerr] = sens_spec(com_phenotypes, M, pathway,'comp', T,P,G);

%------------------------------------------------------------------------------
% -- calculate functional enrichment of each gene module, compare
% -- ontol & automatic modules

fns={'Crosstalk';'Reg (transcr)';'Actuator'; 'Sensor';'Reg (enz)';'Unknown'};
% phenos={'Motile';'Non-motile'};

roles1 = importdata('../db/tigr-roles1.txt');
roles2 = importdata('../db/tigr-roles2.txt');

S1 = clust_composition(T,perm,EO,fns);
S11 = clust_composition(T1,perm,EO,fns);
%print('-depsc', '-cmyk','ontol-to-module.eps');

S2 = clust_composition(T,perm,TGR1,roles1);
S3 = clust_composition(T,perm,TGR2,roles2);

%------------------------------------------------------------------------------
% -- generate controls for entire data set

% random phylogenetic control of all genes in network
if (p=='com-test')
  [mean_rand_genom,std_rand_genom,pval] = ...
      rand_control_mix(ngenes,HIN,SUB,29/62,mean(D));
else
  [mean_rand_genom,std_rand_genom,pval] = ...
      rand_control(ngenes,SUB,mean(D));
end
      
%------------------------------------------------------------------------------
% -- store a bunch of statistics per module
Stats = [];

for i=1:ngeneclusts
  
  ind = find(T==perm(i));		% genes in this module
  n = length(ind);			% size of module
  nedges = (n*(n-1))/2;			% max possible edges
  
  Stats(i,1) = n;
  
  % distances for this module
  d = eudist(M(ind,:));
  m = mean(d);
  Stats(i,2) = m; %squareform(DD(ind,ind)));
  Stats(i,3) = std(d);

  if (n>2)

    % synthetically generated phylo profiles
    %[Stats(i,4),Stats(i,5),foo] = random_control(n,nspecies,[],1,d); 
    %,'synth');
    
    % randomly chosen from whole genome phylo profiles
    [Stats(i,6), Stats(i,7), Stats(i,10)] = rand_control(n, SUB, m);
    
    % randomly chosen from network phylo profiles
    [Stats(i,8),Stats(i,9), Stats(i,11)] = rand_control(n, M, m);
    %,'netwk');
  else
    Stats(i,4:11) = 0;
  end
  
  [frac, majority_fn] = max(S1(:,i));

  % hygepdf(x,m,k,n): suppose you have a lot of m items of which you know k
  % are of a certain type.  what is the probability of drawing x
  % items of that type if you select n at random?
  Stats(i,12) = hygepdf(round(frac*n),ngenes,length(find(EO==majority_fn)),n);

  % now do the same thing, but randomly permuting the labels of the
  % functions, check whether randomly permuted labels are still significant
  FET = [];
  for j=1:1000
    a = randperm(ngenes);
    EOr = EO(a);
    S1r = clust_composition(T,perm,EOr,fns);
    [fracr, majority_fnr] = max(S1r(:,i));
    FET(j) = hygepdf(round(fracr*n),ngenes,length(find(EOr==majority_fnr)),n);
  end

  Stats(i,18) = mean(FET);

  m = MED(ind);
  dn = DN(ind);
  Stats(i,13) = mean(m);
  Stats(i,14) = mean(m)/n;
  Stats(i,15) = mean(dn);
  Stats(i,16) = mean(dn)/n;
  Stats(i,17) = mean(SIL(ind));

end

species_Stats = [];

for i=1:nspeciesclusts
  
  ind = find(T2==perm2(i));		% species in this module
  n = length(ind);			% size of module
  species_Stats(i,1) = perm2(i);
  species_Stats(i,2) = n;
  
  
  [frac, majority_fn] = max(spec_S1(:,i));
  % hygepdf(x,m,k,n): suppose you have a lot of m items of which you know k
  % are of a certain type.  what is the probability of drawing x
  % items of that type if you select n at random?
  species_Stats(i,3) = majority_fn;
  species_Stats(i,4) = hygepdf(round(frac*n),nspecies, ...
			       length(find(phes==majority_fn)),n);
  species_Stats(i,5:12) = unique_phenos(majority_fn,:);
end

%------------------------------------------------------------------------------
 

% print gene stats file
F = fopen([path,pathway,'_module_stats.txt'],'w');
fprintf(F,'# module num, num genes, eucl mean (std), ');
fprintf(F,'rand phylo eudist mean (std), ');
fprintf(F,'rand network eudist mean (std), ');
fprintf(F,'p(phylo eudist diff), p(netwk eudist diff), p(fn signif), ');
fprintf(F,'median (per gene), dnds (per gene), mean silhouette\n');

for i=1:ngeneclusts
fprintf(F,'%d, %d, %.3f (%.3f), %.3f (%.3f), %.2e, %.3f (%.3f), %.2e, %.2e, ',...
	perm(i), Stats(i,1), Stats(i,2), Stats(i,3), ...
	Stats(i,6), Stats(i,7), Stats(i,10),...
	Stats(i,8),Stats(i,9), Stats(i,11),...
	Stats(i,12));

fprintf(F,'%.3f (%.3f), %.3f (%.3f), %.3f\n', ...
	Stats(i,13), Stats(i,14), Stats(i,15), Stats(i,16), Stats(i,17));
end

fprintf(F,'total, %d, %.3f (%.3f), %.3f (%.3f), -, -, ', ...
	ngenes, mean(D), std(D), mean_rand_genom, std_rand_genom);

fprintf(F,'%.2e, -, -, %.3f (%.3f), %.3f (%.3f),%.3f\n', ...
	pval, mean(MED), mean(MED)/ngenes, ...
	mean(DN), mean(DN)/ngenes,mean(SIL));

fclose(F);

% print species stats file
F = fopen([path,'species_module_stats.txt'],'w');
fprintf(F,['# module num, num species, index unique pheno, p(pheno signif)',...
	   ' che,spo,com,ani,pla,ana,extr,gm+\n']);
fprintf(F,['%d, %d, %d, %.2e,',...
	   '%d,%d,%d,%d,%d,%d,%d,%d\n'],...
	species_Stats');
fclose(F);

%------------------------------------------------------------------------------
% -- check goodness of eng ontology, and other ontologies, against
% -- clustering

T_TGR1 = cluster(linkage(pdist(TGR1,'euclidean'),'average'),'cutoff', 0.01);
T_TGR2 = cluster(linkage(pdist(TGR2,'euclidean'),'average'),'cutoff',0.01);
T_COG = cluster(linkage(pdist(COG,'euclidean'),'average'),'cutoff',0.01);
T_COGNUM = cluster(linkage(pdist(COGNUM,'euclidean'),'average'),'cutoff',0.01);

[Z1,foo,bar,bash] = zscores(T,EO);
[Z2,foo,bar,bash] = zscores(T,T_TGR1);
[Z3,foo,bar,bash] = zscores(T,T_TGR2);
[Z4,foo,bar,bash] = zscores(T,T_COG);
[Z5,foo,bar,bash] = zscores(T,T_COGNUM);

%------------------------------------------------------------------------------
% -- cleanup
system(['mv *.eps ',path]);

%clear a F* c1 c2 i j k val x y;
clear a F* i j k val x y;
clear DD C CM WA Mr AM;
save matname;
system(['mv matname.mat ',p,'-cluster.mat']);

% -- the end.
%==============================================================================
