// Ok, we start our classifier thread here
window = self;

importScripts('convnet-min.js');

var itemData = [];
var columns = [];
var updatePending = false;
var featureHash = {};
var featureMax = [];
var featureMin = [];
var globalTimeout;
var unlearnTimeout;

// Respond to the various messages received
onmessage = function(e) {
  var msgType = e.data[0];

  if(msgType === 'init') {
    // Initialization - we store our copy of the item data
    itemData = e.data[1];
    columns = e.data[2];

    // We also need to initialize the classifier
    var idx = 0;

    itemData.forEach(function(item) {
      var myFeatures = extractFeatures(item, columns);
      Object.keys(myFeatures).forEach(function(feature) {
        if(featureHash[feature] === undefined) {
          featureHash[feature] = idx;
          idx++;
        }

        if(featureMin[featureHash[feature]] === undefined || featureMin[featureHash[feature]] > myFeatures[feature]) {
          featureMin[featureHash[feature]] = myFeatures[feature];
        }

        if(featureMax[featureHash[feature]] === undefined || featureMax[featureHash[feature]] < myFeatures[feature]) {
          featureMax[featureHash[feature]] = myFeatures[feature];
        }
      });
    });

    // TODO - Remove debug code
    console.log('CLASSIFIER - Identified features:');
    console.log([featureHash, featureMin, featureMax]);

    initClassifier();

  } else if(msgType === 'train') {
    console.log('WORKER: Received TRAIN message.');
    console.log(e);

    // Train an array of training samples.
    e.data[1].forEach(function(trainItem) {
      // TODO: Remove debug code
      console.log('WORKER: Processing TRAIN request for item ' + trainItem.idx + ' with label ' + trainItem.result + '...');

      // Update existing data
      itemData[trainItem.idx].result = trainItem.result;

      // Train the classifier on the new item
      classifier.addItem(extractFeatures(itemData[trainItem.idx], columns), trainItem.result);
    });

  } else if(msgType === 'unlearn') {
    console.log('WORKER: Received UNLEARN message.');
    // We want to *unlearn* a trained example!
    // This isn't possible with our neural network model, so we need to reinitialize the
    // whole model (sloooooow).

    // First, we remove the training sample from the item set.
    e.data[1].forEach(function(unlearnItem) {
      // TODO: Remove debug code
      console.log('WORKER: Processing UNLEARN request for ' + unlearnItem + '...');

      // Update existing data
      itemData[unlearnItem].result = undefined;
    });

    if(unlearnTimeout) {
      clearTimeout(unlearnTimeout);
    }
    unlearnTimeout = setTimeout(unlearnTimeoutFunction, 10000);
  } else if(msgType === 'iterate') {
    console.log('WORKER: Received ITERATE message.');

    // We want to run a training iteration over all labeled samples we have (reinforces them).
    var relevantIdxes = [];

    itemData.forEach(function(item, idx) {
      if(item.result === 'hit' || item.result === 'noHit') {
        relevantIdxes.push(idx);
      }
    });

    shuffle(relevantIdxes).forEach(function(itemIdx) {
      classifier.addItem(extractFeatures(itemData[itemIdx], columns), itemData[itemIdx].result);
    });

  } else if(msgType === 'update') {
    console.log('WORKER: Received UPDATE message.');

    // We want to update the results for a list of itemData elements.
    updatePercentages(e.data[1]);
  }
};

// Feature to unlearn, to be called using timeout
var unlearnTimeoutFunction = function() {
  // Is another update pending? If yes, reset the timer.
  if(updatePending) {
    console.log('WORKER: Unlearn timeout has fired, but other updates are pending. Waiting 10sec...');
    unlearnTimeout = setTimeout(unlearnTimeoutFunction, 10000);
    return;
  }

  // Otherwise re-initialize the whole classifier.
  console.log('WORKER: Unlearn timeout has fired. Re-initializing classifier...');
  initClassifier();
};

// Re-initializes the neural network and forces it to retrain everything.
var initClassifier = function() {
  // Update is pending...
  updatePending = true;

  // Initialize a new classifier.
  classifier = new neuralNetClassifier(featureHash, featureMin, featureMax);

  // Do we already have items with results? Then we need to train on them.
  itemData.forEach(function(item) {
    if(item.result === 'hit' || item.result === 'noHit') {
      classifier.addItem(extractFeatures(item, columns), item.result);
    }
  });

  console.log('WORKER: Classifier has been re-initialized.');
};

// Update Percentages over all items
var updatePercentages = function(idxes) {
  console.log('WORKER: Starting percentage update.');

  // We are starting update.
  postMessage([[], 'started']);

  percUpdates = [];
  idxes.forEach(function(idx, count) {
    var item_features = extractFeatures(itemData[idx], columns);

    var hit_prob = classifier.probabilityOfClass(item_features, 'hit');
    // var falsepos_prob = classifier.probabilityOfClass(item_features, 'noHit');

    itemData[idx].percentage = Math.round(100000 * hit_prob) / 1000;
    percUpdates.push([idx, Math.round(100000 * hit_prob) / 1000]);
    // TODO: Use prior percentage!

    // TODO - Remove debug code
    console.log('For idx ' + idx + ' we received percentage ' + hit_prob + ' from classifier, using features:');
    console.log(item_features);

    // For every 1000th item, we will post new percentages.
    if(((count+1) % 1000) === 0) {
      console.log('WORKER: Posting interim percentage update (count = ' + count + ')...');
      postMessage([percUpdates, count + ' / ' + idxes.length]);
      percUpdates = [];
    }
  });

  console.log('WORKER: Completed percentage update.');

  // Send message with updated percentages
  postMessage([percUpdates, 'done']);

  // No more updates pending
  updatePending = false;
};



/*
 *  Feature extraction function for extracting features from an item
 */
extractFeatures = function(item, columns) {
  var features = {};

  var addFeature = function(f, val) {
    if(f !== '') {
      if(features[f] === undefined) {
        features[f] = (val ? val : 1);
      } else {
        features[f] += (val ? val : 1);
      }
    }
  };

  // We iterate over all columns
  columns.forEach(function(col) {
    if(col.type === 'number') {
      // For numbers, we just add the number directly as feature
      if(item[col.column] !== undefined && !isNaN(parseFloat(item[col.column]))) {
        addFeature(col.column, parseFloat(item[col.column]));
      } else {
        addFeature(col.column + '_NaN');
      }

    } else if(col.type === 'date') {
      // For dates, we add the date, day of week, day of month, month separately to address
      // potential seasonality
      if(item[col.column] instanceof Date) {
        addFeature(col.column, item[col.column].getTime());
        addFeature(col.column + '_dow_' + item[col.column].getDay());
        addFeature(col.column + '_dom_' + item[col.column].getDate());
        addFeature(col.column + '_month_' + item[col.column].getMonth());
      } else {
        addFeature(col.column + '_noDate');
      }

    } else if(col.type === 'text') {
      // We split texts and add words on each
      // TODO: use stemmer etc., support unicode
      if(item[col.column] !== undefined && item[col.column] !== null && item[col.column] !== '') {
        item[col.column].replace(/[^\w\s]/g, '').split(/\s+/).forEach(function(w) { addFeature(col.column + '_' + w); });
      } else {
        addFeature(col.column + '_empty');
      }

    }

  });

  return features;

  // TODO: we should use better logic here, such as distribution assumptions
};








/************************************************************************************/
/*                                                                                  */
/* NEURAL NETWORK CLASSIFIER                                                        */
/*                                                                                  */
/************************************************************************************/

neuralNetClassifier = function(_featureHash, _featureMin, _featureMax) {
  /*
   * Create a new instance when not using the `new` keyword.
   */
  if (!(this instanceof neuralNetClassifier)) {
    return new neuralNetClassifier(_featureHash);
  }

  this.featureHash = _featureHash;
  this.featureMin = _featureMin;
  this.featureMax = _featureMax;
  this.num_features = Object.keys(_featureHash).length;

  // Features to Array
  this.featuresToVol = function(item_features) {
    var returnVol = new convnetjs.Vol(1, 1, this.num_features, 0.0);

    for(var feature in item_features) {
      var featureIdx = this.featureHash[feature];

      // Normalize features (around 1, as undefined is set to zero by 0/1 indicators)
      returnVol.w[featureIdx] = (item_features[feature]-this.featureMin[featureIdx]+1)/Math.max(this.featureMax[featureIdx]-this.featureMin[featureIdx], 1);
    }

    // TODO: Remove debug code
    // console.log('featuresToVol called:');
    // console.log([item_features, this.featureHash, this.featureMin, this.featureMax, this.num_features, returnVol]);

    return returnVol;
  };


  var layer_defs = [];
  // input layer of size 1x1x2 (all volumes are 3D)
  layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth: this.num_features});
  // some fully connected layers
  layer_defs.push({type:'fc', num_neurons:5, activation:'relu'});
  layer_defs.push({type:'fc', num_neurons:5, activation:'relu'});
  // a softmax classifier predicting probabilities for two classes: 0,1
  layer_defs.push({type:'softmax', num_classes:2});

  this.net = new convnetjs.Net();
  this.net.makeLayers(layer_defs);

  this.trainer = new convnetjs.Trainer(this.net, { method: 'adadelta', l2_decay: 0.000000001, batch_size: 1 });

}


neuralNetClassifier.prototype.addItem = function(item_features, label) {
  var labelNum = (label === 'hit' ? 1 : 0);

  console.log('TRAIN ON:');
  console.log(this.featuresToVol(item_features));
  console.log(labelNum);

  this.trainer.train(this.featuresToVol(item_features), labelNum);
};


neuralNetClassifier.prototype.probabilityOfClass = function(item_features, label) {
  var labelNum = (label === 'hit' ? 1 : 0);
  return this.net.forward(this.featuresToVol(item_features)).w[labelNum];
};



////// Shuffle Algorithm
////// From https://stackoverflow.com/a/2450976

function shuffle(array) {
  var currentIndex = array.length, temporaryValue, randomIndex;

  // While there remain elements to shuffle...
  while (0 !== currentIndex) {

    // Pick a remaining element...
    randomIndex = Math.floor(Math.random() * currentIndex);
    currentIndex -= 1;

    // And swap it with the current element.
    temporaryValue = array[currentIndex];
    array[currentIndex] = array[randomIndex];
    array[randomIndex] = temporaryValue;
  }

  return array;
}