/***************************************************************************
                          main.cpp  -  description
                             -------------------
    begin                : Don Mai  8 18:21:21 CEST 2003
    copyright            : (C) 2003 by klaus frieler
    email                : kgf@omniversum.de
 ***************************************************************************/

/***************************************************************************
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 ***************************************************************************/

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <iostream>
#include <algorithm>
#include <fstream>
#include <cstdlib>
#include <time.h>

#include "EditDistance.cpp"
#include "File.h"
#include "Melody.h"
#include "Fourier.h"
#include "gauss.h"
#include "MetricTree.h"
#include "CmdLine.h"
#include "Types.h"
#include "VectorManip.cpp"
#include "Misc.h"
#include "freq.cpp"
#include "CAutoCorrelation.h"
#include "similarity.h"
#include "chords.h"

#if 0
#include <xercesc/parsers/XercesDOMParser.hpp>
#include <xercesc/dom/DOM.hpp>
#include <xercesc/sax/HandlerBase.hpp>
#include <xercesc/util/XMLString.hpp>
#include <xercesc/util/PlatformUtils.hpp>
XERCES_CPP_NAMESPACE_USE
#endif
using namespace std;

static verbosity  verbLevel = normal;
static string s_sFieldSeparator(";");
static uint s_floatFormat = FLOAT_FORMAT_GERMAN;

#ifndef FEATURE
static const char* getVersion()
{
   return "1.0.10";
}
static void usage()
{

   cerr<<"Usage: simile [OPTIONS] file1 file2               [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -f  fileFile              [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -f1 fileFile              [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -F  fileFile  file        [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -F2 fileFile1 fileFile2   [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -q  fileFile  query       [OUTPUT]"<<endl;
   cerr<<"       simile [OPTIONS] -Q  fileFile  fileFile    [OUTPUT]"<<endl;
   cerr<<"where OPTIONS can be one or more of the following:"<<endl;
   cerr<<"-s,  --silent             silent, no output."<<endl;
   cerr<<"-v,  --verbose            verbose, most output."<<endl;
   cerr<<"-t,  --types TYPESFILE    use TYPESFILE, default is \"types.ini\"."<<endl;
   cerr<<"-th, --threshold THRESH   give extra info if sim>thres."<<endl;   
   cerr<<"-a,  --append             append to output file."<<endl;
   cerr<<"-r,  --random N           choose N random files."<<endl;
   cerr<<"-P,  --phrase             use phrase mode."<<endl;
   cerr<<"-g,  --quant  QUANT       set quantization level to QUANT (as inverse division,"<<endl;
   cerr<<"                          default is 24)."<<endl;
   cerr<<"-d,  --decimal DEC        where DEC can be ',' (default) or '.' "<<endl;
   cerr<<"     --FS SEP             sets output field separator to SEP"<<endl;
   cerr<<"                          default is ';'"<<endl;
   cerr<<"OR"<<endl;
   cerr<<"       simile -p          print all supported similarity types to stdout"<<endl;
   cerr<<"       simile -h          show this help"<<endl;
}
#else
static const char* getVersion()
{
   return "1.0.14";
}
static void usage()
{

   cerr<<"Usage: melfeature [OPTIONS] FEATURES FILES"<<endl;
   cerr<<"where "<<endl;
   cerr<<"      FEATURES = ( (-f | --features) FEATURE [,FEATURE [,...]) | FEATUREFILE  "<<endl;
   cerr<<"      FILES    = ( (-l | --list)     FILEFILE  )    | FILENAME"<<endl<<endl;
   cerr<<"and OPTIONS can be one or more of the following:"<<endl<<endl;
   cerr<<"-m,  --mode MODE          where MODE can be 'f' (feature mode, default)"<<endl; 
   cerr<<"                          or  't' (transform mode)"<<endl;
   cerr<<"-o,  --output FILENAME    use FILENAME as outputfile (resp. as"<<endl;
   cerr<<"                          base name in transform mode)"<<endl;
   cerr<<"-n,  --no-header          write no header line to output"<<endl;
   cerr<<"-g,  --quant  QUANT       set quantization level to QUANT (as inverse division,"<<endl;
   cerr<<"                          default is 24)."<<endl;
   cerr<<"-d,  --decimal DEC        where DEC can be ',' (default) or '.' "<<endl;
   cerr<<"-r,  --R  FORMAT          output format, where FORMAT can be 'csv'"<<endl;
   cerr<<"                          (csv-format, default),'R' (R compatible)"<<endl;
   cerr<<"                          or 'arff' (Weka)"<<endl;
   cerr<<"OR:"<<endl;
   cerr<<"-p,  --print              print list of all features to stdout."<<endl;
   cerr<<"-e,  --explanation        print list of features with short explanations."<<endl;
   cerr<<"-t   --ton   FILE         show Krumhansl tonality analysis of file."<<endl;
   cerr<<"-h,  --help               display this help"<<endl;
}
#endif

static void showSimilarity(const char* text, double val1, double val2)
{
   cerr<<"   "<<text<<" "<<val1<<" ("<<val2<<") "<<endl;
}

static char *substituteChar(char *str, const char from, const char to)
{
   uint len = strlen(str);
   if(!len) return str;

   for(uint i = 0; i<len; ++i){
      if(str[i] == from)
         str[i] = to;
   }
   return str;
}

data evalSimilarities(const Melody &mel1, const Melody &mel, SimTypeList &types, 
                      const int transp, const double thres, const bool bUsePhrases, string &vals, bool &bExceedsThresh)
{
   char buf[128]  = "";
   double val1 = 0, val2 = 0, sum = 0.;
   data values;
   SimTypeList::iterator stl_iter;
   CombiType::iterator   ct_iter;
   SimList::iterator     sl_iter;
   bool bInvalidFlag = false;
   Melody mel2(mel);

   mel2.transpose(transp);

   //sprintf(buf, "%d", transp);
   for(stl_iter = types.begin(); stl_iter != types.end(); ++stl_iter){
      sum = 0.0;
      bInvalidFlag = false;
      if(bUsePhrases){
         sum = mel1.phraseSimilarity(mel2, *stl_iter);
      }
      else{
         sum = mel1.similarity(mel2, *stl_iter, val2);
      }
      uint l = types.size();
      if(stl_iter->size()>1) val2 = 0.0;//not defined for true combi measures!
      sprintf(buf, "%1.4f%s", sum, s_sFieldSeparator.c_str());
      if(s_floatFormat == FLOAT_FORMAT_GERMAN){
         vals += string(substituteChar(buf, '.', ','));
      }else{
         vals += string(buf);
      }
      values.push_back(sum);
      string &combiName = stl_iter->begin()->m_sCombiName;
      
      if(verbLevel>silent){
         showSimilarity(combiName.c_str(), sum, val2);
      }
      if(thres>0.0 && sum>=thres){
         bExceedsThresh = true;
         cerr<<"Found sim>= "<<thres<<" for "<<mel1.getName().c_str()<<":"<<mel2.getName().c_str()<<endl;
         if(verbLevel==silent){
            showSimilarity(combiName.c_str(), sum, val2);
         }
      }
   }
   
   if(verbLevel>silent){
      cerr<<endl<<endl;
   }
   return values;
}

void printStatistics(const dataVec& vals)
{
   data sums, means, sds;
   uint i = 0,j = 0;
   uint N = vals.size();
   uint M = vals[i].size();
   sums = vals[i];
   for(i = 1; i<vals.size();++i){
      for(j = 0; j<M; ++j){
         sums[j] += vals[i][j];
      }
   }
   for(j = 0; j<M; ++j){
      means.push_back(sums[j]/(N+0.0));

   }
   sds.assign(M, 0);
   for(i = 0; i<vals.size();++i){
      for(j = 0; j<M; ++j){
         sds[j] += (means[j] - vals[i][j])*(means[j] - vals[i][j]);
      }
   }
   for(j = 0; j<sds.size(); ++j){
      sds[j]  = N>1 ? sqrt(sds[j])/(N - 1.0) : sqrt(sds[j])/(N);
   }
   cerr<<"N = "<<N<<endl;
   dumpData("Sums", sums, cout);
   dumpData("Means", means, cout);
   dumpData("Standard deviations", sds, cout);
   for(j = 0; j<sds.size(); ++j){
      sds[j] /= means[j];
   }
   dumpData("Standard deviations %", sds, cout);

}
#ifndef FEATURE
static const Option options[] = {
   Option( allfileOpt,  "f",  "all-file",      1),
   Option( nextFileOpt, "f1", "next",          1),
   Option( onefileOpt,  "F",  "one-file",      2),
   Option( filefileOpt, "F2", "file-file",     2),
   Option( silentOpt,   "s",  "silent",        0),
   Option( verboseOpt,  "v",  "verbose",       0),
   Option( typeFileOpt, "t",  "types",         1),
   Option( threshOpt,   "th", "threshold",     1),
   Option( appendOpt,   "a",  "append",        0),
   Option( randomOpt,   "r",  "random",        1),
   Option( queryOpt,    "q",  "query-file",    2),
   Option( queryFullOpt,"Q",  "queries-file",  2),
   Option( dirOpt,      "D",  "directory",     1),
   Option( phraseOpt,   "P",  "phrase",        0),
   Option( fileListOpt, "l",  "list",          0),
   Option( printOpt,    "p",  "print",         0),
   Option( quantOpt,    "g",  "quant",         1),
   Option( fieldSepOpt, "",   "FS"   ,         1),
   Option( floatFormatOpt,    "d", "decimal",  1)
};
#else
static const Option options[] = {
   Option( allfileOpt,        "l", "list",      1),
   Option( typeFileOpt,       "f", "features",  1),
   Option( outputOpt,         "m", "mode",      1),
   Option( outputFileOpt,     "o", "output",    1),
   Option( headerOpt,         "n", "no-header", 0),
   Option( floatFormatOpt,    "d", "decimal",   1),
   Option( tonalityOpt,       "t", "ton",       0),
   Option( outputFormatOpt,   "r", "R",         1),
   Option( printOpt,          "p", "print",     0),
   Option( explanationOpt,    "e", "explanation", 0),
   Option( helpOpt,           "h", "help",      0),
   Option( quantOpt,          "g", "quant",     1)
};
#endif


int showTonality(const char* file, const uint quantLevel)
{
   int bRet = EXIT_FAILURE;
   if(!file) return bRet;
   Melody *pMel = 0;
   vector<string>    fileList;
   File              fileReader;
   //read melody file
   fileList.push_back(string(file));
   fileReader.readMelodies(&pMel, fileList, quantLevel);
   if(fileList.empty()) return bRet;
   pMel->setOStream(&cout);
   pMel->printInfo();
   data dummy;
   cout<<"Barwise analysis: "<<endl;
   cout<<"----------------- "<<endl;
   for(uint i = 0; i<pMel->getBarCount(); ++i){
      pMel->getRawTonalityKrumhansl(i+1, dummy, true);
   }
   
   cout<<"Total Key analysis: "<<endl;
   cout<<"--------------------"<<endl;
   pMel->getRawTonalityKrumhansl(-1, dummy, true);
   cout<<endl;


   data vec;
   cout<<"Barwise tonality vectors (Major)"<<endl;
   cout<<"-------------------------------"<<endl;
   for(uint i = 0; i<pMel->getBarCount(); ++i){
      vec = pMel->getRawTonality(i+1, MAJOR, true);
   }
   cout<<endl;
   cout<<"Barwise tonality vectors (Minor)"<<endl;
   cout<<"-------------------------------"<<endl;
   for(uint i = 0; i<pMel->getBarCount(); ++i){
      pMel->getRawTonality(i+1, MINOR, true);
   }
   cout<<endl;
   cout<<"Total tonality vectors (Major)"<<endl;
   cout<<"-----------------------------"<<endl;
   vec = pMel->getRawTonality(0, MAJOR, true);
   cout<<endl;
   cout<<"Total tonality vectors (Minor)"<<endl;
   cout<<"-----------------------------"<<endl;
   pMel->getRawTonality(0, MINOR,true);

   return EXIT_SUCCESS;
}
#ifndef FEATURE
int main(int argc, char *argv[])
{
   bool              bNeedGauss = false;
   bool              fileFlag  = false;
   bool              nextFileFlag = false;
   string            sSimResult(""), sResultFile(""), szTypeFile("types.ini"), header(""), sFullPath("");
   Melody            mel1, mel2;
   File              file;

   SimTypeList       SimTypes;
   vector<string>    fileList1, fileList2;
   double            thresh = -1.0;
   data              values, sums;
   dataVec           totVals;
   uint              N = 0;
   uint              fileMode = FILE_MODE_WRITE;
   uint              random = 0;
   uint              quantLevel = 24;
   bool              queryMode = false;
   bool              bPhraseMode = false;
   bool              bWriteFileList = false;
   clock_t start = clock();
   CmdLine cmdLine(options, sizeof(options)/sizeof(options[0]), argc, argv);
   cmdLine.parse();

   if(cmdLine.hasOption(printOpt)){
      file.printSimTypes();
      return EXIT_SUCCESS;
   }

   if(cmdLine.hasOption(verboseOpt)) verbLevel = verbose;
   else if(cmdLine.hasOption(silentOpt)) verbLevel = silent;
   else verbLevel = normal;
   file.setVerbosity(verbLevel);
   if(cmdLine.hasOption(threshOpt)){
      if(!cmdLine.getFloatParam(threshOpt, 0,thresh)){
         usage();
         exit(-1);
      }
   }
   if(cmdLine.hasOption(appendOpt)){
      fileMode = FILE_MODE_APPEND;
   }
   if(cmdLine.hasOption(phraseOpt)){
      bPhraseMode = true;
   }
   if(cmdLine.hasOption(fileListOpt)){
      bWriteFileList = true;
   }
   if(cmdLine.hasOption(randomOpt)){
      if(!cmdLine.getUIntParam(randomOpt, 0, random)){
         usage();
         exit(-1);
      }
   }
   if(cmdLine.hasOption(quantOpt)){
      if(!cmdLine.getUIntParam(quantOpt, 0, quantLevel)){
         usage();
         exit(-1);
      }
      if(ggT(quantLevel,2) != 2 && ggT(quantLevel,3) !=3){
         cerr<<"Warning: You're using a non-standard quantization level ("<<quantLevel<<")"<<endl;
      }
   }

   if(cmdLine.hasOption(dirOpt)){
      if(!cmdLine.getStringParam(dirOpt, 0, sFullPath)){
         usage();
         exit(-1);
      }
   }
   if(cmdLine.hasOption(fieldSepOpt)){
      if(!cmdLine.getStringParam(fieldSepOpt, 0, s_sFieldSeparator)){
         usage();
         exit(-1);
      }
   }
   file.setFieldSeparator(s_sFieldSeparator);
   //read name of types file
   if(cmdLine.hasOption(typeFileOpt)){
      if(!cmdLine.getStringParam(typeFileOpt, 0, szTypeFile )){
         usage();
         exit(-1);
      }
   }
   if(cmdLine.hasOption(floatFormatOpt)){
      string sParam;
      if(!cmdLine.getStringParam(floatFormatOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      if(sParam[0] == ','){
         s_floatFormat = FLOAT_FORMAT_GERMAN;
      }
      else if(sParam[0] == '.'){
         s_floatFormat = FLOAT_FORMAT_ENGLISH;
      }
      else{
         usage();
         exit(-1);
      }

   }  

   //read one or more file files
   string sFile ="";
   if(cmdLine.hasOption(filefileOpt)){
      if(!cmdLine.getStringParam(filefileOpt, 0, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         return -1;
       }
      if(!cmdLine.getStringParam(filefileOpt, 1, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList2)){
         exit(-1);
      }      
   }
   else if(cmdLine.hasOption(queryFullOpt)){
      if(!cmdLine.getStringParam(queryFullOpt, 0, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         return -1;
       }
      if(!cmdLine.getStringParam(queryFullOpt, 1, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList2)){
         exit(-1);
      }
      queryMode = true;
      verbLevel = silent;
   }
   else if(cmdLine.hasOption(onefileOpt)){
      if(!cmdLine.getStringParam(onefileOpt, 0, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         exit(-1);
       }
      if(!cmdLine.getStringParam(onefileOpt, 1, sFile)){
         usage();
         exit(-1);
      }        
      fileList2.push_back(sFile);
   }
   else if(cmdLine.hasOption(queryOpt)){
      if(!cmdLine.getStringParam(queryOpt, 0, sFile)){
         usage();
         exit(-1);
      }
      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         exit(-1);
       }
      if(!cmdLine.getStringParam(queryOpt, 1, sFile)){
         usage();
         exit(-1);
      }
      fileList2.push_back(sFile);
      queryMode = true; 
      //verbLevel = silent;
   }
   else if(cmdLine.hasOption(allfileOpt)){
      if(!cmdLine.getStringParam(allfileOpt, 0,sFile)){
         usage();
         exit(-1);
      }
      
      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         return -1;
      }
      fileList2 = fileList1;
      fileFlag = true;
   }
   else if(cmdLine.hasOption(nextFileOpt)){
      if(!cmdLine.getStringParam(nextFileOpt, 0, sFile)){
         usage();
         exit(-1);
      }

      if(!file.readFilesFile(sFile.c_str(), fileList1)){
         return -1;
      }

      fileList2 = fileList1;
      fileFlag = true;
      nextFileFlag = true;
   }
   else {
      if(!cmdLine.getFreeParam(0, sFile)){
         usage();
         exit(-1);
      }
      fileList1.push_back(sFile);
      if(!cmdLine.getFreeParam(1, sFile)){
         usage();
         exit(-1);
      }
      fileList2.push_back(sFile);
   }
   
   if(cmdLine.hasOption(-1)){
      int count = cmdLine.getFreeParamCount();
      if
      (   count == 3
      || (count == 1 &&
         (cmdLine.hasOption(queryOpt)
          ||cmdLine.hasOption(onefileOpt) 
          ||cmdLine.hasOption(allfileOpt)
          || cmdLine.hasOption(filefileOpt)
          || cmdLine.hasOption(nextFileOpt)
          || cmdLine.hasOption(queryFullOpt)
          )
         )
      )
      {
         if(!cmdLine.getFreeParam(count-1, sResultFile)) {
            usage();
            exit(-1);
         }
      }
   }
   //read types file
   if(!file.readTypesFile(szTypeFile.c_str(), SimTypes, header, bNeedGauss)){
      file.getDefaultSimTypes(SimTypes, header, bNeedGauss);
   }
   if(SimTypes.empty() ){
      cerr<<"No valid similarity measure found"<<endl;
      exit(-1);
   }
   //if random option prepare fileLists
   if(random != 0){
      srand(time(0));
      fileList1 = getRandomSelection(fileList1, random);
      if(!fileFlag){
         fileList2 = getRandomSelection(fileList2, random);
      }
      else{
         fileList2 = fileList1;
      }
      //dumpData("File1:", fileList1);
      //dumpData("File2:", fileList2);
   }
   //init gauss if necessary
   if(bNeedGauss)
      initGauss();

   Melody *pMel1 = 0;
   Melody *pMel2 = 0;

   //read all melody files into arrays
   file.readMelodies(&pMel1, fileList1, quantLevel);
   if(!fileFlag){
      file.readMelodies(&pMel2, fileList2, quantLevel);   
   }
   else {
      pMel2 = pMel1;
   }
   cerr<<"Reading files completed in "<<(clock()-start)*1000.0/CLOCKS_PER_SEC<<" ms"<<endl;

   //preamble
   if(verbLevel>silent)
      cerr<<"simile v"<<getVersion()<<" (c) 2003-2006 Klaus Frieler & Daniel Muellensiefen"<<endl<<endl;
   int nComp = fileFlag ? fileList1.size()*(fileList1.size()+1)/2 : fileList1.size()*fileList2.size();
   if(nextFileFlag){
      nComp = fileList1.size()-1;
   }
   if(bPhraseMode){
      uint i = 0, j = 0;
      ofstream phrase("phraseInfo.txt");
      for(i = 0; i<fileList1.size(); ++i){
         if(pMel1[i].empty()) continue;
         pMel1[i].printInfo(phrase);
         MelodyList l = pMel1[i].getPhrases();
         for(j = 0; j<l.size(); ++j){
            l[j]->printInfo(phrase);
         }
      }
      if(!fileFlag){
         for(i = 0; i<fileList2.size(); ++i){
            if(pMel2[i].empty()) continue;
            pMel2[i].printInfo(phrase);
            MelodyList l = pMel2[i].getPhrases();
            for(j = 0; j<l.size(); ++j){
               l[j]->printInfo(phrase);
            }
         }         
      }
   }

   
   if(verbLevel>silent){
      cerr<<"Number of files 1: "<<fileList1.size()<<endl;
      cerr<<"Number of files 2: "<<fileList2.size()<<endl;
      cerr<<"Number of comparings: "<<nComp<<endl<<endl;
   }

   //write header if neccessary
   if(sResultFile.length() && fileMode == FILE_MODE_WRITE && !queryMode){
      file.printCSV(sResultFile.c_str(), fileMode, header.c_str());
   }

   //compare all file in fileList1 with all in fileList2,
   //if identic eval just the half due to symmetry   
   for(uint i = 0; i<fileList1.size(); ++i){
      
      uint jStart = fileFlag ? i+1 : 0;
      if(jStart>fileList2.size()) continue;

      uint jEnd = nextFileFlag ? jStart +1 : fileList2.size();
      if(jEnd>fileList2.size()) jEnd = fileList2.size();
      
      for(uint j = jStart; j<jEnd; ++j){
         //if(pMel1[i].empty() || pMel2[j].empty()) continue;
         if(verbLevel>silent){
            cerr<<"Similarities "<<fileList1[i].c_str()<<":"<<fileList2[j].c_str()<<endl;
            cerr<<"----------------------------------------------------"<<endl<<endl;
            if(verbLevel>normal){
               if(!pMel1[i].empty()){
                  pMel1[i].printInfo();
                  cerr<<endl;
               }
               if(!pMel2[j].empty()){
                  pMel2[j].printInfo();
                  cerr<<endl;
               }
            }
         }
         //get best transposition
         clock_t start3 = clock();
         char bestTrans = pMel1[i].getBestTransposition(pMel2[j]);
         if(verbLevel>normal){
            cerr<<"Best transposition found in "<<(clock()-start3)*1000.0/CLOCKS_PER_SEC<<" ms"<<endl;
         }
         if(verbLevel>normal){
            cerr<<"Best transposition: "<<(int)bestTrans<<" semitones of "<<fileList2[j].c_str()<<endl;
         }
         sSimResult = string(fileList1[i]) + s_sFieldSeparator + string(fileList2[j].c_str()) + s_sFieldSeparator;
         values.clear();
         N++;
         bool bExceedsThresh = thresh>0.0 ? false : true;
         clock_t start2 = clock();
         values = evalSimilarities(pMel1[i], pMel2[j], SimTypes, bestTrans, thresh, bPhraseMode, sSimResult, bExceedsThresh);
         if(verbLevel>normal){
            cerr<<"Similarity computation completed in "<<(clock()-start2)*1000.0/CLOCKS_PER_SEC<<" ms"<<endl;
         }
         totVals.push_back(values);
         if(!queryMode && sResultFile.length() && bExceedsThresh)
            file.printCSV(sResultFile.c_str(), FILE_MODE_APPEND, sSimResult.c_str());
      }         

   }
#if 0
   cout<<"#nexus"<<endl<<endl;
   cout<<"BEGIN Taxa;"<<endl;
   cout<<"DIMENSIONS ntax="<<fileList1.size()<<";"<<endl;
   cout<<"TAXLABELS"<<endl;
   for(uint i = 0; i< fileList1.size(); ++i){
      cout<<"["<<i<<"]"<<" '"<<fileList1[i].c_str()<<"' "<<endl;
   }
   cout<<";"<<endl<<"END; [Taxa]"<<endl;
   cout<<endl;
   cout<<"BEGIN Distances;"<<endl;
   cout<<"DIMENSIONS ntax="<<fileList1.size()<<";"<<endl;
   cout<<"FORMAT labels=left diagonal triangle=both;"<<endl;
   cout<<"MATRIX"<<endl; 
   for(uint i = 0; i< fileList1.size(); ++i){

      cout<<"["<<i<<"]"<<" '"<<fileList1[i].c_str()<<"' ";
      for(uint j = 0; j<fileList2.size(); ++j){
         double val = (totVals[i*fileList2.size()+j])[0];
         if(val<=0) val= 2;
         else val = -log(val);
         cout<<val<<" ";
      }
      cout<<endl;
   }
   cout<<";"<<endl<<"END; [Distances]"<<endl;
#endif
   if(queryMode){      
      file.writeQueryResultFile(sResultFile, fileList1, fileList2, totVals, sFullPath, -1, SimTypes);
   }

   //printStatistics(totVals);
   if(bNeedGauss)
      exitGauss();

   if(pMel1 != pMel2){
      if(pMel2){
         delete[] pMel2;
         pMel2 = 0;
      }
   }

   if(pMel1){
      delete[] pMel1;
      pMel1 = 0;
   }
   if(bWriteFileList){
      FILE *fp = fopen("filelist.dat", "w");
      if(fp){
         for(uint i = 0; i< fileList1.size(); ++i){
            fprintf(fp, "%d:%s\n", i, fileList1[i].c_str());         
         }      
      }
      fclose(fp);
   }
   cerr<<nComp<<" comparings completed in "<<(clock()-start)*1.0/CLOCKS_PER_SEC<<" s"<<endl;
   return EXIT_SUCCESS;
}

#else
//here goes melfeature...

void testSimilarity()
{
   CSimilarity<unsigned char> *sim1 = 0;
   CSimilarity<double> *sim2 = 0;
   CSimilarity<char> *sim3 = 0;
   CSimilarity<char> *sim4 = 0;
   double sigma = 2;   
   ucharSimFactory.Create(&sim1, eSimType_Delta);
   doubleSimFactory.Create(&sim2, eSimType_Mult);
   charSimFactory.Create(&sim3, eSimType_Gauss, &sigma);
   charSimFactory.Create(&sim4, eSimType_Delta);

   CSimPair<unsigned char, double> test(sim1, sim2);
   pair<unsigned char, double> v1(0x0f, 34.23);
   pair<unsigned char, double> v2(0xf0, 42.31);
   double sim = test(v1, v2);
   cout<<"Similarity = "<<sim<<endl;
   string s("ABACABADEEFFEEFFABACABADEEFFEEFF");
   const CAutoCorrelation<char> autoC;
   vector<double> t = autoC.Calculate(stringToVector(s), *sim4, true);
   dumpData("auto = ", t, cout);
   const CAutoCorrelation<double> autoD;
   vector<double> t2 = autoD.Calculate(t, *sim2, true);
   dumpData("auto = ", t2, cout);
   vector<double> diff = sub<double>(t, t2);
   dumpData("diff = ", diff, cout);
}

void printMetricSimilarities()
{
   initGauss();
   double sigmas [] = {0.5, 0.25, 0.125, 0.05, 0.025, 0.0125, 0.001, 0.0001};
   FILE * fp, *fp1 = 0;
   fp1 = fopen("riemann.dat", "wb");
   for(uint i = 2; i<=12; ++i){
      char fname[32] ="";
      sprintf(fname, "metric%d.dat", i);
      fp = fopen(fname , "wb");
      if(!fp) continue;
      for(uint j = i; j<=12; ++j){
         double sim = metricSimilarity(i,j, 0.001, 1);
         printf("Sim(N=%d, M = %d) = %f\n", i,j,sim);
         fprintf(fp, "%d %f\n", j, sim);
         if(j<=2*i){
            fprintf(fp1, "%f %f\n", (j+.0)/i, sim);
         }
      }
     fclose(fp);
   }
}

void dataToBuf(char *buf, size_t cbSize, const char* delim, const data& vec, 
               const uint floatFormat = FLOAT_FORMAT_GERMAN, const char* text = 0 )
{
   if(!buf || vec.empty()) return;
   stringstream str, str1;
   uint textLen = 0;
   if(text){
      str<<text<<delim;
      textLen = strlen(text) + 1;
   }
   for(uint i = 0; i<vec.size(); ++i){
      str1<<vec[i];
      if(i!=vec.size()-1) str1<<delim;
   }
   char *sTmp = new char[cbSize];
   if(!sTmp){
      cerr<<"Out of memory!"<<endl;
      return;
   }
   str1.get(sTmp, cbSize-textLen);
   if(floatFormat == FLOAT_FORMAT_GERMAN){
      subchr(sTmp, '.', ',');
   }
   str<<sTmp;
   str.get(buf, cbSize);
   delete sTmp;
}
string getARFFHeader(const string& firstColumn, const FeatureList &features)
{
   size_t j = 0;
   string header = string("@relation melodic-features\n\n");
   //header += string("@attribute ") + firstColumn + string(" string\n");
   for(j = 0; j<features.size(); ++j){
      header += string("@attribute ") + features[j]->getName().c_str() + string(" numeric\n");
   }
   header +=  string("@data\n\n");    
   return header;
}

string getHeader(const string& firstColumn, uint outputFormat, const FeatureList &features)
{
   size_t j = 0;
   if(outputFormat == OUTPUT_FORMAT_ARFF) 
      return getARFFHeader(firstColumn, features);

   const char* delim = outputFormat == OUTPUT_FORMAT_CSV ? ";" : " ";
   string header = firstColumn +delim;
   for(j = 0; j<features.size(); ++j){
      header += features[j]->getName().c_str();
      if(j!=features.size()-1) header += delim;
   }
     
   return header;
}

bool writeFeatures(const FeatureList &features, const vector<string>&  fileList, const string &sResultFile, 
                   const bool useHeader, const uint nFloatFormat, const uint outputFormat, const uint quantLevel)
{
   bool bRet = false;
   char buf[4096]="";
   Melody *pMel = 0;
   File file;

   //read all melody files into array
   if(!file.readMelodies(&pMel, fileList, quantLevel)) return false;

   size_t i = 0, j = 0;
   FILE *fp = 0;
   if(sResultFile.empty()){
      fp = stdout;
   }
   else{
      fp = fopen(sResultFile.c_str(), "w");
      if(!fp) return false;
   }
   uint nFFormat = outputFormat == OUTPUT_FORMAT_CSV ?  s_floatFormat : FLOAT_FORMAT_ENGLISH; 
   const char *delim    = outputFormat != OUTPUT_FORMAT_CSV ?  (outputFormat == OUTPUT_FORMAT_ARFF ? "," : " ") : ";"; 
   if(useHeader){
      string header = getHeader("file", outputFormat, features);
      fprintf(fp, "%s\n", header.c_str());
      if(ferror(fp)) goto out;
   }   
   for(i = 0; i<fileList.size(); ++i){
      if(pMel[i].empty()) continue;
      data tmp;
      for(j = 0; j<features.size(); ++j){
         data feature = pMel[i].getFeature(*features[j]);
         if(!feature.empty())tmp.push_back(feature[0]);
      }
      dataToBuf(buf, sizeof(buf), delim, tmp, nFFormat);
      if(outputFormat != OUTPUT_FORMAT_ARFF){
         fprintf(fp, "%s%s%s\n", fileList[i].c_str(), delim, buf);
      }
      else{
         fprintf(fp, "%s\n", buf);
      }
      if(ferror(fp)) goto out;
   }
   
   bRet = true;
out:
   if(pMel){
      delete[] pMel;
      pMel = 0;
   }
   if(fp!=stdout)fclose(fp);
   return bRet;
}

string getTransformationFileName(const string &fileName, const string &sResult)
{
   string baseName = stripExtension(fileName.c_str());
   string ret;
   if(!sResult.empty()){
      ret = baseName + "_"+ sResult + ".csv";
   }
   else{
      ret = baseName + "_trans.csv";   
   }
   return ret;
}
void getTransformationMatrix(const Melody &melody, const FeatureList &features, dataVec& ret)
{
   dataVec tmp;
   size_t maxSize = 0; 
   size_t i, j = 0;
   data pitches = melody.getPitchesAsDouble();
   tmp.push_back(pitches);
   for(j = 0; j<features.size(); ++j){
      data feature = melody.getFeature(*features[j]);
      tmp.push_back(feature);
      if(feature.size() > maxSize) maxSize = feature.size();
   }
   //transpose matrix of feature vectors
   for( j = 0; j <maxSize; ++j){
      data row;
      for(i = 0; i<tmp.size(); ++i){
         if(j<tmp[i].size()){
            row.push_back(tmp[i][j]);
         }
      }
      ret.push_back(row);
   }
}
bool writeTransformation(const FeatureList &features, const vector<string>&  fileList, const string sResultFile, 
                         const bool useHeader, const uint nFloatFormat, const uint outputFormat, const uint quantLevel)
{
   bool bRet = false;
   char buf[4096]="";
   Melody *pMel = 0;
   File file;

   //read all melody files into array
   if(!file.readMelodies(&pMel, fileList, quantLevel)) return false;

   size_t i = 0, j = 0;

   for(i = 0; i<fileList.size(); ++i){
      if(pMel[i].empty()) continue;
      
      FILE *fp = 0;
      if(sResultFile.empty()){
         fp = stdout;
      }
      else{
         string sFile = getTransformationFileName(fileList[i], sResultFile); 
         fp = fopen(sFile.c_str(), "w");
         if(fp == NULL) continue;
      }
      uint nFFormat = outputFormat == OUTPUT_FORMAT_CSV ?  s_floatFormat : FLOAT_FORMAT_ENGLISH; 
      const char *delim    = outputFormat == OUTPUT_FORMAT_CSV ?  ";" : " "; 
      if(useHeader){
         string header = getHeader("pitch", outputFormat, features);
         fprintf(fp, "file%s%s\n", delim, header.c_str());
         if(ferror(fp)){
            if(fp != stdout) fclose(fp);
            continue;
         }
      }

      dataVec transMatrix;
      getTransformationMatrix(pMel[i], features, transMatrix);
      for(j = 0; j<transMatrix.size(); ++j){
         dataToBuf(buf, sizeof(buf), delim, transMatrix[j], nFFormat, fileList[i].c_str());
         fprintf(fp, "%s\n", buf);
         if(ferror(fp)){
            break;
         }
      }
      if(fp != stdout) fclose(fp);

   }

   if(pMel){
      delete[] pMel;
      pMel = 0;
   }
   return bRet;
}
void testChordAnalyzer()
{
   PitchSet ps;
   PitchSetPreprocessor psp;
   /*
   for(uint i = 60; i<72; ++i){
      ps.push_back(i);
   }
      */

   ps.push_back(60);
   //ps.push_back(62);
   //ps.push_back(64);
   ps.push_back(65);
   //ps.push_back(67);
   //ps.push_back(69);
   ps.push_back(70);
   psp.Init(ps);
   string str = psp.ToString();
   cout<<str<<endl;
   ChordsAnalyzer ca(0);
   ca.Analyze(psp.GetWeightedPitchSet());
}
int main(int argc, char *argv[])
{
   string            sResultFile(""), szFeatureFile(""), header(""), sFullPath("");
   Melody            mel;
   File              file;

   FeatureList       features;
   vector<string>    fileList;
   uint              fileMode = FILE_MODE_WRITE;
   uint              random = 0;
   uint              mode = OUTPUT_MODE_FEATURE;
   bool              bUseHeader = true;
   //uint              nFloatFormat = FLOAT_FORMAT_GERMAN;
   uint              outputFormat = OUTPUT_FORMAT_CSV;
   uint              quantLevel = 24;
   data vec;
   //testSimilarity();
   //return 0;
   //testChordAnalyzer();
   //return 0;
   //printMetricSimilarities();
   //return 0;
   //parse command line
   //MetricTree2 mtest(atoi(argv[1]));
   //mtest.dump();
   //return 0;
   CmdLine cmdLine(options, sizeof(options)/sizeof(options[0]), argc, argv);
   cmdLine.parse();
   //cmdLine.dumpOptions();
   if(cmdLine.hasOption(helpOpt)){
      usage();
      return EXIT_SUCCESS;
   }
   if(cmdLine.hasOption(printOpt)){
      file.printFeatures(false);
      return EXIT_SUCCESS;
   }
   if(cmdLine.hasOption(explanationOpt)){
      file.printFeatures(true);
      return EXIT_SUCCESS;
   }
   string sFile ="";
   if(cmdLine.hasOption(tonalityOpt)){
      if(!cmdLine.getStringParam(tonalityOpt, 0, sFile)){
         usage();
         exit(-1);
      }
      return showTonality(sFile.c_str(), 24);
   }

   //get verbosity   
   if(cmdLine.hasOption(verboseOpt)) verbLevel = verbose;
   else if(cmdLine.hasOption(silentOpt)) verbLevel = silent;
   else verbLevel = normal;

   if(cmdLine.hasOption(headerOpt)) bUseHeader = false;

   if(cmdLine.hasOption(randomOpt)){
      if(!cmdLine.getUIntParam(randomOpt, 0, random)){
         usage();
         exit(-1);
      }
   }
   if(cmdLine.hasOption(quantOpt)){
      if(!cmdLine.getUIntParam(quantOpt, 0, quantLevel)){
         usage();
         exit(-1);
      }
   }

   if(cmdLine.hasOption(outputFileOpt)){
      if(!cmdLine.getStringParam(outputFileOpt, 0, sResultFile)){
         usage();
         exit(-1);
      }
   }

   string sParam="";
   if(cmdLine.hasOption(outputOpt)){
      if(!cmdLine.getStringParam(outputOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      if(sParam[0] == 'f'){
         mode = OUTPUT_MODE_FEATURE;
      }
      else if(sParam[0] == 't'){
         mode = OUTPUT_MODE_TRANSFORM;
      }
      else{
         usage();
         exit(-1);
      }

   }  
   if(cmdLine.hasOption(outputFormatOpt)){
      if(!cmdLine.getStringParam(outputFormatOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      int param = toupper(sParam[0]);
      if(param == 'C'){
         outputFormat = OUTPUT_FORMAT_CSV;
      }
      else if(param == 'R'){
         outputFormat = OUTPUT_FORMAT_R;
      }
      else if(param == 'A'){
         outputFormat = OUTPUT_FORMAT_ARFF;
      }
      else if(param == 'G'){
         outputFormat = OUTPUT_FORMAT_GNU;
      }
      else{
         usage();
         exit(-1);
      }

   }  
   if(cmdLine.hasOption(floatFormatOpt)){
      if(!cmdLine.getStringParam(floatFormatOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      if(sParam[0] == ','){
         s_floatFormat = FLOAT_FORMAT_GERMAN;
      }
      else if(sParam[0] == '.'){
         s_floatFormat = FLOAT_FORMAT_ENGLISH;
      }
      else{
         usage();
         exit(-1);
      }

   }  

   //read name of feature file
   if(cmdLine.hasOption(typeFileOpt)){
      if(!cmdLine.getStringParam(typeFileOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      //read feature file
      if(!file.getFeaturesFromString(sParam, features)){
         if(!file.readFeatureFile(sParam.c_str(), features)){
            usage();
            exit(-1);
         }
      }
   }
   else{
      usage();
      exit(-1);      
   }

   //read one or more file files
   if(cmdLine.hasOption(allfileOpt)){
      if(!cmdLine.getStringParam(allfileOpt, 0, sParam)){
         usage();
         exit(-1);
      }
      
      if(!file.readFilesFile(sParam.c_str(), fileList)){
         exit(-1);
      }
   }
   else {
      if(!cmdLine.getFreeParam(0, sParam)){
         usage();
         exit(-1);
      }
      fileList.push_back(sParam);

   }
   //if random option prepare fileLists
   if(random != 0){
      srand(time(0));
      fileList = getRandomSelection(fileList, random);
   }

 
   //preamble
   cerr<<"melfeature v"<<getVersion()<<" (c) 2003-2008 Klaus Frieler & Daniel Muellensiefen"<<endl<<endl;

   if(verbLevel>silent){
      cerr<<"Number of files: "<<fileList.size()<<endl;
   }
   bool bRet  = false;
   clock_t start = clock();
   switch(mode){
      case OUTPUT_MODE_FEATURE:
         bRet = writeFeatures(features, fileList, sResultFile, bUseHeader, s_floatFormat, outputFormat, quantLevel);
         break;
      case OUTPUT_MODE_TRANSFORM:
         bRet = writeTransformation(features, fileList, sResultFile, bUseHeader, s_floatFormat, outputFormat, quantLevel);
         break;
   }
   
   cerr<<"Task completed in "<<(clock()-start)*1.0/CLOCKS_PER_SEC<<" s"<<endl;
   return bRet ? EXIT_SUCCESS : EXIT_FAILURE;
}

#endif