From 61ced60acff41f0ca44b44ecd2766e7c39f0bd56 Mon Sep 17 00:00:00 2001 From: sjyan Date: Tue, 7 Nov 2017 11:42:40 +0800 Subject: [PATCH 01/11] Update to Keras 2.0.8 --- nea/models.py | 6 +++--- nea/my_layers.py | 6 ++++-- nea/w2vEmbReader.py | 16 +++++++++------- train_nea.py | 11 ++++++++--- 4 files changed, 24 insertions(+), 15 deletions(-) diff --git a/nea/models.py b/nea/models.py index c80b758..3db0cb8 100644 --- a/nea/models.py +++ b/nea/models.py @@ -70,10 +70,10 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): model.add(Dense(num_outputs)) if not args.skip_init_bias: bias_value = (np.log(initial_mean_value) - np.log(1 - initial_mean_value)).astype(K.floatx()) - model.layers[-1].b.set_value(bias_value) + model.layers[-1].bias = bias_value model.add(Activation('sigmoid')) model.emb_index = 0 - + elif args.model_type == 'breg': logger.info('Building a BIDIRECTIONAL REGRESSION model') from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge @@ -130,7 +130,7 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing lookup table') emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - model.layers[model.emb_index].W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].W.get_value())) + model.layers[model.emb_index].set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].get_weights())) logger.info(' Done') return model diff --git a/nea/my_layers.py b/nea/my_layers.py index 9855fd7..01805d2 100644 --- a/nea/my_layers.py +++ b/nea/my_layers.py @@ -54,11 +54,12 @@ def __init__(self, mask_zero=True, **kwargs): def call(self, x, mask=None): if self.mask_zero: - return K.cast(x.sum(axis=1) / mask.sum(axis=1, keepdims=True), K.floatx()) + mask = K.cast(mask, K.floatx()) + return K.cast(K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims= True), K.floatx()) else: return K.mean(x, axis=1) - def get_output_shape_for(self, input_shape): + def compute_output_shape(self, input_shape): return (input_shape[0], input_shape[2]) def compute_mask(self, x, mask): @@ -73,6 +74,7 @@ class Conv1DWithMasking(Convolution1D): def __init__(self, **kwargs): self.supports_masking = True super(Conv1DWithMasking, self).__init__(**kwargs) + def compute_mask(self, x, mask): return mask diff --git a/nea/w2vEmbReader.py b/nea/w2vEmbReader.py index 04f0d52..e09a255 100644 --- a/nea/w2vEmbReader.py +++ b/nea/w2vEmbReader.py @@ -28,9 +28,10 @@ def __init__(self, emb_path, emb_dim=None): counter = 0 for line in emb_file: tokens = line.split() - assert len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info' + # modified by Shengjia Yan @2017-11-03 Friday word = tokens[0] - vec = tokens[1:] + vec = tokens[1].split(',') + assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' self.embeddings[word] = vec counter += 1 assert counter == self.vocab_size, 'Vocab size does not match the header info' @@ -41,13 +42,14 @@ def __init__(self, emb_path, emb_dim=None): self.embeddings = {} for line in emb_file: tokens = line.split() + word = tokens[0] + vec = tokens[1].split(',') if self.emb_dim == -1: - self.emb_dim = len(tokens) - 1 + self.emb_dim = len(vec) assert self.emb_dim == emb_dim, 'The embeddings dimension does not match with the requested dimension' else: - assert len(tokens) == self.emb_dim + 1, 'The number of dimensions does not match the header info' - word = tokens[0] - vec = tokens[1:] + assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' + self.embeddings[word] = vec self.vocab_size += 1 @@ -63,7 +65,7 @@ def get_emb_matrix_given_vocab(self, vocab, emb_matrix): counter = 0. for word, index in vocab.iteritems(): try: - emb_matrix[index] = self.embeddings[word] + emb_matrix[0][index] = self.embeddings[word] counter += 1 except KeyError: pass diff --git a/train_nea.py b/train_nea.py index acd06bb..5a6f995 100755 --- a/train_nea.py +++ b/train_nea.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import os import argparse import logging import numpy as np @@ -9,6 +10,8 @@ import nea.utils as U import pickle as pk + +os.environ['KERAS_BACKEND']='theano' logger = logging.getLogger(__name__) ############################################################################################################################### @@ -164,9 +167,9 @@ ## Plotting model # -from keras.utils.visualize_util import plot +#from keras.utils.visualize_util import plot -plot(model, to_file = out_dir + '/model.png') +#plot(model, to_file = out_dir + '/model.png') ############################################################################################################################### ## Save model architecture @@ -189,14 +192,16 @@ logger.info('--------------------------------------------------------------------------------------------------------------------------') logger.info('Initial Evaluation:') -evl.evaluate(model, -1, print_info=True) +#evl.evaluate(model, -1, print_info=True) total_train_time = 0 total_eval_time = 0 + for ii in range(args.epochs): # Training t0 = time() + train_history = model.fit(train_x, train_y, batch_size=args.batch_size, nb_epoch=1, verbose=0) tr_time = time() - t0 total_train_time += tr_time From 26efaad42708943d8f912d7288fdf650e5f61931 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 15:17:58 +0800 Subject: [PATCH 02/11] nea 1.0 --- README.md | 11 +++- nea/__init__.py | 5 ++ nea/asap_evaluator.py | 111 ++++++++++++++++++++++++++++++-- nea/asap_reader.py | 47 +++++++++++--- nea/models.py | 12 +++- nea/my_kappa_calculator.py | 6 ++ nea/my_layers.py | 12 ++-- nea/optimizers.py | 6 ++ nea/plot_confusion_matrix.py | 98 ++++++++++++++++++++++++++++ nea/quadratic_weighted_kappa.py | 6 +- nea/utils.py | 6 ++ nea/w2vEmbReader.py | 24 +++++-- train_nea.py | 25 +++---- 13 files changed, 329 insertions(+), 40 deletions(-) mode change 100644 => 100755 README.md mode change 100644 => 100755 nea/__init__.py mode change 100644 => 100755 nea/asap_evaluator.py mode change 100644 => 100755 nea/asap_reader.py mode change 100644 => 100755 nea/models.py mode change 100644 => 100755 nea/my_kappa_calculator.py mode change 100644 => 100755 nea/my_layers.py mode change 100644 => 100755 nea/optimizers.py create mode 100644 nea/plot_confusion_matrix.py mode change 100644 => 100755 nea/quadratic_weighted_kappa.py mode change 100644 => 100755 nea/utils.py mode change 100644 => 100755 nea/w2vEmbReader.py diff --git a/README.md b/README.md old mode 100644 new mode 100755 index b11739f..d36b4a6 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Neural Essay Assessor # +# Neural Essay Assessor 1.0 # An automatic essay scoring system based on convolutional and recurrent neural networks, including GRU and LSTM. @@ -8,6 +8,14 @@ An automatic essay scoring system based on convolutional and recurrent neural ne * Prepare data * Run train_nea.py +### Environment + +- Keras 1.1.0 +- Theano 0.8.2 +- Python 2.7.9 +- Numpy 1.13.3 +- Scipy 0.19.1 + ### Data ### We have used 5-fold cross validation on ASAP dataset to evaluate our system. This dataset (training_set_rel3.tsv) can be downloaded from [here](https://www.kaggle.com/c/asap-aes/data). After downloading the file, put it in the [data](https://github.com/nusnlp/nea/tree/master/data) directory and create training, development and test data using ```preprocess_asap.py``` script: @@ -47,6 +55,7 @@ Neural Essay Assessor is licensed under the GNU General Public License Version 3 * Kaveh Taghipour (kaveh@comp.nus.edu.sg) * Hwee Tou Ng (nght@comp.nus.edu.sg) +* Shengjia Yan (i@yanshengjia.com) ### Publication ### diff --git a/nea/__init__.py b/nea/__init__.py old mode 100644 new mode 100755 index e69de29..fdb558e --- a/nea/__init__.py +++ b/nea/__init__.py @@ -0,0 +1,5 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com \ No newline at end of file diff --git a/nea/asap_evaluator.py b/nea/asap_evaluator.py old mode 100644 new mode 100755 index fba1045..dd74115 --- a/nea/asap_evaluator.py +++ b/nea/asap_evaluator.py @@ -1,20 +1,33 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + +import sys +reload(sys) +sys.setdefaultencoding('utf8') + from scipy.stats import pearsonr, spearmanr, kendalltau import logging import numpy as np +import itertools +import matplotlib.pyplot as plt from nea.my_kappa_calculator import quadratic_weighted_kappa as qwk from nea.my_kappa_calculator import linear_weighted_kappa as lwk +from sklearn.metrics import confusion_matrix logger = logging.getLogger(__name__) class Evaluator(): - def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, dev_y_org, test_y_org): + def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, train_y_org, dev_y_org, test_y_org): self.dataset = dataset self.prompt_id = prompt_id self.out_dir = out_dir self.dev_x, self.test_x = dev_x, test_x - self.dev_y, self.test_y = dev_y, test_y - self.dev_y_org, self.test_y_org = dev_y_org, test_y_org + self.dev_y, self.test_y = dev_y, test_y # 标准化后的人工打分 [0, 1] + self.train_y_org, self.dev_y_org, self.test_y_org = train_y_org, dev_y_org, test_y_org # 原始的人工打分 self.dev_mean = self.dev_y_org.mean() self.test_mean = self.test_y_org.mean() self.dev_std = self.dev_y_org.std() @@ -27,15 +40,101 @@ def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, de self.batch_size = 180 self.low, self.high = self.dataset.get_score_range(self.prompt_id) self.dump_ref_scores() + self.generate_contrast_result(self.best_dev_epoch) def dump_ref_scores(self): + logger.info('Saving reference scores') + np.savetxt(self.out_dir + '/preds/train_ref.txt', self.train_y_org, fmt='%i') np.savetxt(self.out_dir + '/preds/dev_ref.txt', self.dev_y_org, fmt='%i') np.savetxt(self.out_dir + '/preds/test_ref.txt', self.test_y_org, fmt='%i') + logger.info(' Done') def dump_predictions(self, dev_pred, test_pred, epoch): np.savetxt(self.out_dir + '/preds/dev_pred_' + str(epoch) + '.txt', dev_pred, fmt='%.8f') np.savetxt(self.out_dir + '/preds/test_pred_' + str(epoch) + '.txt', test_pred, fmt='%.8f') - + + # modified by sjyan @2017-10-26 + def plot_confusion_matrix(self, cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): + """ + This function prints and plots the confusion matrix. + Normalization can be applied by setting `normalize=True`. + """ + if normalize: + # find out how many samples per class have received their correct label + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + + # get the precision (fraction of class-k predictions that have ground truth label k) + # cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + print("Normalized confusion matrix") + else: + print('Confusion matrix, without normalization') + + print(cm) + + plt.imshow(cm, interpolation='nearest', cmap=cmap) + plt.title(title) + plt.colorbar() + tick_marks = np.arange(len(classes)) + plt.xticks(tick_marks, classes, rotation=45) + plt.yticks(tick_marks, classes) + + fmt = '.2f' if normalize else 'd' + thresh = cm.max() / 2. + for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): + plt.text(j, i, format(cm[i, j], fmt), + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black") + + plt.tight_layout() + plt.ylabel('True label') + plt.xlabel('Predicted label') + + # modified by sjyan @2017-10-24 + def generate_contrast_result(self, best_dev_epoch): + logger.info('Generating contrast result') + dev_ref_contrast_path = self.out_dir + '/preds/dev_ref_contrast.txt' + test_ref_contrast_path = self.out_dir + '/preds/test_ref_contrast.txt' + + essays = [] + ref_score = [] # true label + pred_score = [] # prediction label + diff_1_counter = 0 + diff_2_counter = 0 + + essays_file = open(self.out_dir + '/preds/dev_essays.txt', 'r') + ref_file = open(self.out_dir + '/preds/dev_ref.txt', 'r') + pred_file = open(self.out_dir + '/preds/dev_pred_49.txt', 'r') + contrast_result_file = open(dev_ref_contrast_path, 'a') + contrast_result_file.seek(0) + contrast_result_file.truncate() + contrast_result_file.write('ref_score(2-12) pred_score essay\n') + + for essay in essays_file.readlines(): + essay = essay.strip('\n') + essays.append(essay) + for ref in ref_file.readlines(): + ref = ref.strip('\n') + ref_score.append(ref) + for pred in pred_file.readlines(): + pred = pred.strip('\n') + pred_score.append(pred) + for i in range(len(essays)): + ref_t = float(ref_score[i]) + pred_t = float(pred_score[i]) + if abs(ref_t - pred_t) <= 2: + diff_2_counter += 1 + if abs(ref_t - pred_t) <= 1: + diff_1_counter += 1 + string = ref_score[i] + ' ' + pred_score[i] + ' ' + essays[i] + '\n' + contrast_result_file.write(string) + + essay_counter = len(essays) + contrast_result_file.write('\nTotal number of essays: %d\n' % essay_counter) + contrast_result_file.write('diff <= 1 : %d (%.2f%%)\n' % (diff_1_counter, 100 * diff_1_counter / essay_counter)) + contrast_result_file.write('diff <= 2 : %d (%.2f%%)\n' % (diff_2_counter, 100 * diff_2_counter / essay_counter)) + # contrast_result_file.write('Pearson: 0.824\n') # TODO + logger.info(' Done') + def calc_correl(self, dev_pred, test_pred): dev_prs, _ = pearsonr(dev_pred, self.dev_y_org) test_prs, _ = pearsonr(test_pred, self.test_y_org) @@ -59,14 +158,16 @@ def evaluate(self, model, epoch, print_info=False): self.dev_loss, self.dev_metric = model.evaluate(self.dev_x, self.dev_y, batch_size=self.batch_size, verbose=0) self.test_loss, self.test_metric = model.evaluate(self.test_x, self.test_y, batch_size=self.batch_size, verbose=0) + # normalized score self.dev_pred = model.predict(self.dev_x, batch_size=self.batch_size).squeeze() self.test_pred = model.predict(self.test_x, batch_size=self.batch_size).squeeze() + # unnormalized score self.dev_pred = self.dataset.convert_to_dataset_friendly_scores(self.dev_pred, self.prompt_id) self.test_pred = self.dataset.convert_to_dataset_friendly_scores(self.test_pred, self.prompt_id) self.dump_predictions(self.dev_pred, self.test_pred, epoch) - + self.dev_prs, self.test_prs, self.dev_spr, self.test_spr, self.dev_tau, self.test_tau = self.calc_correl(self.dev_pred, self.test_pred) self.dev_qwk, self.test_qwk, self.dev_lwk, self.test_lwk = self.calc_qwk(self.dev_pred, self.test_pred) diff --git a/nea/asap_reader.py b/nea/asap_reader.py old mode 100644 new mode 100755 index f2a9731..4d04a7a --- a/nea/asap_reader.py +++ b/nea/asap_reader.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import random import codecs import sys @@ -7,6 +13,9 @@ import numpy as np import pickle as pk +reload(sys) +sys.setdefaultencoding('utf8') + logger = logging.getLogger(__name__) num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') ref_scores_dtype = 'int32' @@ -140,22 +149,35 @@ def read_essays(file_path, prompt_id): essays_ids.append(int(tokens[0])) return essays_list, essays_ids -def read_dataset(file_path, prompt_id, maxlen, vocab, tokenize_text, to_lower, score_index=6, char_level=False): +def read_dataset(file_path, output_path, prompt_id, maxlen, vocab, tokenize_text, to_lower, score_index=6, char_level=False): logger.info('Reading dataset from: ' + file_path) + + # modified by sjyan @2017-10-25 + essays = [] + + if 'train' in file_path: + essays_path = output_path + '/preds/train_essays.txt' + if 'dev' in file_path: + essays_path = output_path + '/preds/dev_essays.txt' + if 'test' in file_path: + essays_path = output_path + '/preds/test_essays.txt' + if maxlen > 0: logger.info(' Removing sequences with more than ' + str(maxlen) + ' words') data_x, data_y, prompt_ids = [], [], [] num_hit, unk_hit, total = 0., 0., 0. maxlen_x = -1 with codecs.open(file_path, mode='r', encoding='UTF8') as input_file: - input_file.next() + input_file.next() for line in input_file: tokens = line.strip().split('\t') essay_id = int(tokens[0]) essay_set = int(tokens[1]) - content = tokens[2].strip() + content = tokens[2].strip() # 作文内容 score = float(tokens[score_index]) - if essay_set == prompt_id or prompt_id <= 0: + if essay_set == prompt_id or prompt_id <= 0: # 如果 作文所属集合编号 等于 命题集编号 + # modified by sjyan @2017-10-25 + essays.append(content) if to_lower: content = content.lower() if char_level: @@ -188,11 +210,17 @@ def read_dataset(file_path, prompt_id, maxlen, vocab, tokenize_text, to_lower, s if maxlen_x < len(indices): maxlen_x = len(indices) logger.info(' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total)) + + # modified by sjyan @2017-10-25 + logger.info('Saving essays to: ' + essays_path) + np.savetxt(essays_path, essays, fmt='%s') + logger.info(' total number of essays: %d' % len(essays)) + return data_x, data_y, prompt_ids, maxlen_x -def get_data(paths, prompt_id, vocab_size, maxlen, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=None, score_index=6): +def get_data(paths, prompt_id, vocab_size, maxlen, output_path, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=None, score_index=6): train_path, dev_path, test_path = paths[0], paths[1], paths[2] - + if not vocab_path: vocab = create_vocab(train_path, prompt_id, maxlen, vocab_size, tokenize_text, to_lower) if len(vocab) < vocab_size: @@ -205,9 +233,10 @@ def get_data(paths, prompt_id, vocab_size, maxlen, tokenize_text=True, to_lower= logger.warning('The vocabualry includes %i words which is different from given: %i' % (len(vocab), vocab_size)) logger.info(' Vocab size: %i' % (len(vocab))) - train_x, train_y, train_prompts, train_maxlen = read_dataset(train_path, prompt_id, maxlen, vocab, tokenize_text, to_lower) - dev_x, dev_y, dev_prompts, dev_maxlen = read_dataset(dev_path, prompt_id, 0, vocab, tokenize_text, to_lower) - test_x, test_y, test_prompts, test_maxlen = read_dataset(test_path, prompt_id, 0, vocab, tokenize_text, to_lower) + # modified by sjyan @2017-10-25 + train_x, train_y, train_prompts, train_maxlen = read_dataset(train_path, output_path, prompt_id, maxlen, vocab, tokenize_text, to_lower) + dev_x, dev_y, dev_prompts, dev_maxlen = read_dataset(dev_path, output_path, prompt_id, 0, vocab, tokenize_text, to_lower) + test_x, test_y, test_prompts, test_maxlen = read_dataset(test_path, output_path, prompt_id, 0, vocab, tokenize_text, to_lower) overal_maxlen = max(train_maxlen, dev_maxlen, test_maxlen) diff --git a/nea/models.py b/nea/models.py old mode 100644 new mode 100755 index 3db0cb8..d4193e5 --- a/nea/models.py +++ b/nea/models.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import numpy as np import logging @@ -70,10 +76,10 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): model.add(Dense(num_outputs)) if not args.skip_init_bias: bias_value = (np.log(initial_mean_value) - np.log(1 - initial_mean_value)).astype(K.floatx()) - model.layers[-1].bias = bias_value + model.layers[-1].b.set_value(bias_value) model.add(Activation('sigmoid')) model.emb_index = 0 - + elif args.model_type == 'breg': logger.info('Building a BIDIRECTIONAL REGRESSION model') from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge @@ -130,7 +136,7 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing lookup table') emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - model.layers[model.emb_index].set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].get_weights())) + model.layers[model.emb_index].W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].W.get_value())) logger.info(' Done') return model diff --git a/nea/my_kappa_calculator.py b/nea/my_kappa_calculator.py old mode 100644 new mode 100755 index dd7310b..4f68cc3 --- a/nea/my_kappa_calculator.py +++ b/nea/my_kappa_calculator.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import numpy as np from quadratic_weighted_kappa import quadratic_weighted_kappa as qwk from quadratic_weighted_kappa import linear_weighted_kappa as lwk diff --git a/nea/my_layers.py b/nea/my_layers.py old mode 100644 new mode 100755 index 01805d2..91707e9 --- a/nea/my_layers.py +++ b/nea/my_layers.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import keras.backend as K from keras.engine.topology import Layer from keras.layers.convolutional import Convolution1D @@ -54,12 +60,11 @@ def __init__(self, mask_zero=True, **kwargs): def call(self, x, mask=None): if self.mask_zero: - mask = K.cast(mask, K.floatx()) - return K.cast(K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims= True), K.floatx()) + return K.cast(x.sum(axis=1) / mask.sum(axis=1, keepdims=True), K.floatx()) else: return K.mean(x, axis=1) - def compute_output_shape(self, input_shape): + def get_output_shape_for(self, input_shape): return (input_shape[0], input_shape[2]) def compute_mask(self, x, mask): @@ -74,7 +79,6 @@ class Conv1DWithMasking(Convolution1D): def __init__(self, **kwargs): self.supports_masking = True super(Conv1DWithMasking, self).__init__(**kwargs) - def compute_mask(self, x, mask): return mask diff --git a/nea/optimizers.py b/nea/optimizers.py old mode 100644 new mode 100755 index 556eb53..099fcce --- a/nea/optimizers.py +++ b/nea/optimizers.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import keras.optimizers as opt def get_optimizer(args): diff --git a/nea/plot_confusion_matrix.py b/nea/plot_confusion_matrix.py new file mode 100644 index 0000000..5531deb --- /dev/null +++ b/nea/plot_confusion_matrix.py @@ -0,0 +1,98 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-26 +# Email: i@yanshengjia.com + +import sys +reload(sys) +sys.setdefaultencoding('utf8') + +import logging +import numpy as np +import itertools +import matplotlib.pyplot as plt +from sklearn.metrics import confusion_matrix + + +def load_confusion_matrix(ref_score, pred_score): + ref_file = open('../output/emb/rnn/prompt_1/fold_0/preds/dev_ref.txt', 'r') + pred_file = open('../output/emb/rnn/prompt_1/fold_0/preds/dev_pred_49.txt', 'r') + + for ref in ref_file.readlines(): + ref = ref.strip('\n') + ref = int(ref) + ref_score.append(ref) + for pred in pred_file.readlines(): + pred = pred.strip('\n') + pred = float(pred) + pred = round(pred) + pred_score.append(pred) + +def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): + """ + This function prints and plots the confusion matrix. + Normalization can be applied by setting `normalize=True`. + """ + if normalize: + # 1. find out how many samples per class have received their correct label + # 计算真正类别为k的样本被预测成各个类别的比例 + # e.g. 有25个样本的 true label 是 6，其中10个样本被预测为类别7，那么在混淆矩阵中 true label = 6 并且 predicted label = 7 的一个格子中的值为 0.4 + cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] + + # 2. get the precision (fraction of class-k predictions that have ground truth label k) + # 计算预测的准确率 + # e.g. 预测为类别k的有12个，但其中只有9个的真正类别是k，那么准确率为 0.75 + # cm = cm.astype('float') / cm.sum(axis=0)[:, np.newaxis] + + print("Normalized confusion matrix") + else: + print('Confusion matrix, without normalization') + + print(cm) + + plt.imshow(cm, interpolation='nearest', cmap=cmap) + plt.title(title) + plt.colorbar() + tick_marks = np.arange(len(classes)) + plt.xticks(tick_marks, classes, rotation=45) + plt.yticks(tick_marks, classes) + + fmt = '.2f' if normalize else 'd' + thresh = cm.max() / 2. + for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): + plt.text(j, i, format(cm[i, j], fmt), + horizontalalignment="center", + color="white" if cm[i, j] > thresh else "black") + + # plt.tight_layout() + plt.ylabel('True label') + plt.xlabel('Predicted label') + +def main(): + ref_score = [] # true label + pred_score = [] # predicted label + + load_confusion_matrix(ref_score, pred_score) + + nea_matrix = confusion_matrix(ref_score, pred_score) + np.set_printoptions(precision=2) + class_names = ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] + + # Plot non-normalized confusion matrix + plt.figure() + plot_confusion_matrix(nea_matrix, classes=class_names, title='Confusion matrix, without normalization') + plt.savefig('./unnormalized_cm.png') + + # Plot normalized confusion matrix + plt.figure() + plot_confusion_matrix(nea_matrix, classes=class_names, normalize=True, title='Normalized confusion matrix') + plt.savefig('./normalized_cm.png') + + # plt.show() + +if __name__ == '__main__': + main() + + + diff --git a/nea/quadratic_weighted_kappa.py b/nea/quadratic_weighted_kappa.py old mode 100644 new mode 100755 index 7609840..23d5184 --- a/nea/quadratic_weighted_kappa.py +++ b/nea/quadratic_weighted_kappa.py @@ -1,4 +1,8 @@ -#! /usr/bin/env python2.7 +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com import numpy as np diff --git a/nea/utils.py b/nea/utils.py old mode 100644 new mode 100755 index 6201f1c..37ef24f --- a/nea/utils.py +++ b/nea/utils.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import sys import os, errno import logging diff --git a/nea/w2vEmbReader.py b/nea/w2vEmbReader.py old mode 100644 new mode 100755 index e09a255..5007c94 --- a/nea/w2vEmbReader.py +++ b/nea/w2vEmbReader.py @@ -1,3 +1,9 @@ +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com + import codecs import logging import numpy as np @@ -28,10 +34,12 @@ def __init__(self, emb_path, emb_dim=None): counter = 0 for line in emb_file: tokens = line.split() - # modified by Shengjia Yan @2017-11-03 Friday + # modified by sjyan @2017-10-19 + # deal with comma word = tokens[0] - vec = tokens[1].split(',') - assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' + str = tokens[1] + vec = str.split(',') + assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' self.embeddings[word] = vec counter += 1 assert counter == self.vocab_size, 'Vocab size does not match the header info' @@ -42,14 +50,18 @@ def __init__(self, emb_path, emb_dim=None): self.embeddings = {} for line in emb_file: tokens = line.split() + # modified by sjyan @2017-10-19 + # deal with comma word = tokens[0] - vec = tokens[1].split(',') + str = tokens[1] + vec = str.split(',') + if self.emb_dim == -1: self.emb_dim = len(vec) assert self.emb_dim == emb_dim, 'The embeddings dimension does not match with the requested dimension' else: assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' - + self.embeddings[word] = vec self.vocab_size += 1 @@ -65,7 +77,7 @@ def get_emb_matrix_given_vocab(self, vocab, emb_matrix): counter = 0. for word, index in vocab.iteritems(): try: - emb_matrix[0][index] = self.embeddings[word] + emb_matrix[index] = self.embeddings[word] counter += 1 except KeyError: pass diff --git a/train_nea.py b/train_nea.py index 5a6f995..9af3423 100755 --- a/train_nea.py +++ b/train_nea.py @@ -1,4 +1,8 @@ -#!/usr/bin/env python +# !/usr/bin/python +# -*- coding:utf-8 -*- +# Author: Shengjia Yan +# Date: 2017-10-19 +# Email: i@yanshengjia.com import os import argparse @@ -10,7 +14,6 @@ import nea.utils as U import pickle as pk - os.environ['KERAS_BACKEND']='theano' logger = logging.getLogger(__name__) @@ -29,7 +32,7 @@ parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='rmsprop', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=rmsprop)") parser.add_argument("-l", "--loss", dest="loss", type=str, metavar='', default='mse', help="Loss function (mse|mae) (default=mse)") parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=50, help="Embeddings dimension (default=50)") -parser.add_argument("-c", "--cnndim", dest="cnn_dim", type=int, metavar='', default=0, help="CNN output dimension. '0' means no CNN layer (default=0)") +parser.add_argument("-c", "--cnndim", dest="cnn_dim", type=int, metavar='', default=10, help="CNN output dimension. '0' means no CNN layer (default=0)") parser.add_argument("-w", "--cnnwin", dest="cnn_window_size", type=int, metavar='', default=3, help="CNN window size. (default=3)") parser.add_argument("-r", "--rnndim", dest="rnn_dim", type=int, metavar='', default=300, help="RNN dimension. '0' means no RNN layer (default=300)") parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=32, help="Batch size (default=32)") @@ -72,8 +75,9 @@ from keras.preprocessing import sequence # data_x is a list of lists +# modified by sjyan @2017-10-25 (train_x, train_y, train_pmt), (dev_x, dev_y, dev_pmt), (test_x, test_y, test_pmt), vocab, vocab_size, overal_maxlen, num_outputs = dataset.get_data( - (args.train_path, args.dev_path, args.test_path), args.prompt_id, args.vocab_size, args.maxlen, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=args.vocab_path) + (args.train_path, args.dev_path, args.test_path), args.prompt_id, args.vocab_size, args.maxlen, args.out_dir_path, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=args.vocab_path) # Dump vocab with open(out_dir + '/vocab.pkl', 'wb') as vocab_file: @@ -131,6 +135,7 @@ logger.info(' train_y mean: %s, stdev: %s, MFC: %s' % (str(train_mean), str(train_std), str(mfs_list))) # We need the dev and test sets in the original scale for evaluation +train_y_org = train_y.astype(dataset.get_ref_dtype()) dev_y_org = dev_y.astype(dataset.get_ref_dtype()) test_y_org = test_y.astype(dataset.get_ref_dtype()) @@ -167,9 +172,9 @@ ## Plotting model # -#from keras.utils.visualize_util import plot +from keras.utils.visualize_util import plot -#plot(model, to_file = out_dir + '/model.png') +plot(model, to_file = out_dir + '/model.png') ############################################################################################################################### ## Save model architecture @@ -179,12 +184,12 @@ with open(out_dir + '/model_arch.json', 'w') as arch: arch.write(model.to_json(indent=2)) logger.info(' Done') - + ############################################################################################################################### ## Evaluator # -evl = Evaluator(dataset, args.prompt_id, out_dir, dev_x, test_x, dev_y, test_y, dev_y_org, test_y_org) +evl = Evaluator(dataset, args.prompt_id, out_dir, dev_x, test_x, dev_y, test_y, train_y_org, dev_y_org, test_y_org) ############################################################################################################################### ## Training @@ -192,16 +197,14 @@ logger.info('--------------------------------------------------------------------------------------------------------------------------') logger.info('Initial Evaluation:') -#evl.evaluate(model, -1, print_info=True) +evl.evaluate(model, -1, print_info=True) total_train_time = 0 total_eval_time = 0 - for ii in range(args.epochs): # Training t0 = time() - train_history = model.fit(train_x, train_y, batch_size=args.batch_size, nb_epoch=1, verbose=0) tr_time = time() - t0 total_train_time += tr_time From 1a849bb8034f182cf27cbb2346deb4a0e7084139 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 15:30:40 +0800 Subject: [PATCH 03/11] Delete something unnecessary --- nea/asap_evaluator.py | 91 ------------------------------------------- 1 file changed, 91 deletions(-) diff --git a/nea/asap_evaluator.py b/nea/asap_evaluator.py index dd74115..6b5ef94 100755 --- a/nea/asap_evaluator.py +++ b/nea/asap_evaluator.py @@ -39,101 +39,10 @@ def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, tr self.best_test_missed_epoch = -1 self.batch_size = 180 self.low, self.high = self.dataset.get_score_range(self.prompt_id) - self.dump_ref_scores() - self.generate_contrast_result(self.best_dev_epoch) - - def dump_ref_scores(self): - logger.info('Saving reference scores') - np.savetxt(self.out_dir + '/preds/train_ref.txt', self.train_y_org, fmt='%i') - np.savetxt(self.out_dir + '/preds/dev_ref.txt', self.dev_y_org, fmt='%i') - np.savetxt(self.out_dir + '/preds/test_ref.txt', self.test_y_org, fmt='%i') - logger.info(' Done') def dump_predictions(self, dev_pred, test_pred, epoch): np.savetxt(self.out_dir + '/preds/dev_pred_' + str(epoch) + '.txt', dev_pred, fmt='%.8f') np.savetxt(self.out_dir + '/preds/test_pred_' + str(epoch) + '.txt', test_pred, fmt='%.8f') - - # modified by sjyan @2017-10-26 - def plot_confusion_matrix(self, cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): - """ - This function prints and plots the confusion matrix. - Normalization can be applied by setting `normalize=True`. - """ - if normalize: - # find out how many samples per class have received their correct label - cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] - - # get the precision (fraction of class-k predictions that have ground truth label k) - # cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] - print("Normalized confusion matrix") - else: - print('Confusion matrix, without normalization') - - print(cm) - - plt.imshow(cm, interpolation='nearest', cmap=cmap) - plt.title(title) - plt.colorbar() - tick_marks = np.arange(len(classes)) - plt.xticks(tick_marks, classes, rotation=45) - plt.yticks(tick_marks, classes) - - fmt = '.2f' if normalize else 'd' - thresh = cm.max() / 2. - for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): - plt.text(j, i, format(cm[i, j], fmt), - horizontalalignment="center", - color="white" if cm[i, j] > thresh else "black") - - plt.tight_layout() - plt.ylabel('True label') - plt.xlabel('Predicted label') - - # modified by sjyan @2017-10-24 - def generate_contrast_result(self, best_dev_epoch): - logger.info('Generating contrast result') - dev_ref_contrast_path = self.out_dir + '/preds/dev_ref_contrast.txt' - test_ref_contrast_path = self.out_dir + '/preds/test_ref_contrast.txt' - - essays = [] - ref_score = [] # true label - pred_score = [] # prediction label - diff_1_counter = 0 - diff_2_counter = 0 - - essays_file = open(self.out_dir + '/preds/dev_essays.txt', 'r') - ref_file = open(self.out_dir + '/preds/dev_ref.txt', 'r') - pred_file = open(self.out_dir + '/preds/dev_pred_49.txt', 'r') - contrast_result_file = open(dev_ref_contrast_path, 'a') - contrast_result_file.seek(0) - contrast_result_file.truncate() - contrast_result_file.write('ref_score(2-12) pred_score essay\n') - - for essay in essays_file.readlines(): - essay = essay.strip('\n') - essays.append(essay) - for ref in ref_file.readlines(): - ref = ref.strip('\n') - ref_score.append(ref) - for pred in pred_file.readlines(): - pred = pred.strip('\n') - pred_score.append(pred) - for i in range(len(essays)): - ref_t = float(ref_score[i]) - pred_t = float(pred_score[i]) - if abs(ref_t - pred_t) <= 2: - diff_2_counter += 1 - if abs(ref_t - pred_t) <= 1: - diff_1_counter += 1 - string = ref_score[i] + ' ' + pred_score[i] + ' ' + essays[i] + '\n' - contrast_result_file.write(string) - - essay_counter = len(essays) - contrast_result_file.write('\nTotal number of essays: %d\n' % essay_counter) - contrast_result_file.write('diff <= 1 : %d (%.2f%%)\n' % (diff_1_counter, 100 * diff_1_counter / essay_counter)) - contrast_result_file.write('diff <= 2 : %d (%.2f%%)\n' % (diff_2_counter, 100 * diff_2_counter / essay_counter)) - # contrast_result_file.write('Pearson: 0.824\n') # TODO - logger.info(' Done') def calc_correl(self, dev_pred, test_pred): dev_prs, _ = pearsonr(dev_pred, self.dev_y_org) From bd272f0300dd081c83ed380ca2018bf996eff530 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 15:41:01 +0800 Subject: [PATCH 04/11] NEA 2.0 --- README.md | 10 ++-- nea/__init__.py | 5 -- nea/asap_evaluator.py | 30 ++++------ nea/asap_reader.py | 47 +++------------- nea/models.py | 13 ++--- nea/my_kappa_calculator.py | 6 -- nea/my_layers.py | 13 ++--- nea/optimizers.py | 6 -- nea/plot_confusion_matrix.py | 98 --------------------------------- nea/quadratic_weighted_kappa.py | 6 +- nea/utils.py | 6 -- nea/w2vEmbReader.py | 26 +++------ train_nea.py | 25 ++++----- 13 files changed, 52 insertions(+), 239 deletions(-) mode change 100755 => 100644 README.md mode change 100755 => 100644 nea/__init__.py mode change 100755 => 100644 nea/asap_evaluator.py mode change 100755 => 100644 nea/asap_reader.py mode change 100755 => 100644 nea/models.py mode change 100755 => 100644 nea/my_kappa_calculator.py mode change 100755 => 100644 nea/my_layers.py mode change 100755 => 100644 nea/optimizers.py delete mode 100644 nea/plot_confusion_matrix.py mode change 100755 => 100644 nea/quadratic_weighted_kappa.py mode change 100755 => 100644 nea/utils.py mode change 100755 => 100644 nea/w2vEmbReader.py diff --git a/README.md b/README.md old mode 100755 new mode 100644 index d36b4a6..f8d2524 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Neural Essay Assessor 1.0 # +# Neural Essay Assessor 2.0 # An automatic essay scoring system based on convolutional and recurrent neural networks, including GRU and LSTM. @@ -10,11 +10,10 @@ An automatic essay scoring system based on convolutional and recurrent neural ne ### Environment -- Keras 1.1.0 -- Theano 0.8.2 +- Keras 2.0.8 +- Theano 0.9.0 +- Tensorflow 1.3.0 - Python 2.7.9 -- Numpy 1.13.3 -- Scipy 0.19.1 ### Data ### @@ -55,7 +54,6 @@ Neural Essay Assessor is licensed under the GNU General Public License Version 3 * Kaveh Taghipour (kaveh@comp.nus.edu.sg) * Hwee Tou Ng (nght@comp.nus.edu.sg) -* Shengjia Yan (i@yanshengjia.com) ### Publication ### diff --git a/nea/__init__.py b/nea/__init__.py old mode 100755 new mode 100644 index fdb558e..e69de29 --- a/nea/__init__.py +++ b/nea/__init__.py @@ -1,5 +0,0 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com \ No newline at end of file diff --git a/nea/asap_evaluator.py b/nea/asap_evaluator.py old mode 100755 new mode 100644 index 6b5ef94..fba1045 --- a/nea/asap_evaluator.py +++ b/nea/asap_evaluator.py @@ -1,33 +1,20 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - -import sys -reload(sys) -sys.setdefaultencoding('utf8') - from scipy.stats import pearsonr, spearmanr, kendalltau import logging import numpy as np -import itertools -import matplotlib.pyplot as plt from nea.my_kappa_calculator import quadratic_weighted_kappa as qwk from nea.my_kappa_calculator import linear_weighted_kappa as lwk -from sklearn.metrics import confusion_matrix logger = logging.getLogger(__name__) class Evaluator(): - def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, train_y_org, dev_y_org, test_y_org): + def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, dev_y_org, test_y_org): self.dataset = dataset self.prompt_id = prompt_id self.out_dir = out_dir self.dev_x, self.test_x = dev_x, test_x - self.dev_y, self.test_y = dev_y, test_y # 标准化后的人工打分 [0, 1] - self.train_y_org, self.dev_y_org, self.test_y_org = train_y_org, dev_y_org, test_y_org # 原始的人工打分 + self.dev_y, self.test_y = dev_y, test_y + self.dev_y_org, self.test_y_org = dev_y_org, test_y_org self.dev_mean = self.dev_y_org.mean() self.test_mean = self.test_y_org.mean() self.dev_std = self.dev_y_org.std() @@ -39,11 +26,16 @@ def __init__(self, dataset, prompt_id, out_dir, dev_x, test_x, dev_y, test_y, tr self.best_test_missed_epoch = -1 self.batch_size = 180 self.low, self.high = self.dataset.get_score_range(self.prompt_id) + self.dump_ref_scores() + + def dump_ref_scores(self): + np.savetxt(self.out_dir + '/preds/dev_ref.txt', self.dev_y_org, fmt='%i') + np.savetxt(self.out_dir + '/preds/test_ref.txt', self.test_y_org, fmt='%i') def dump_predictions(self, dev_pred, test_pred, epoch): np.savetxt(self.out_dir + '/preds/dev_pred_' + str(epoch) + '.txt', dev_pred, fmt='%.8f') np.savetxt(self.out_dir + '/preds/test_pred_' + str(epoch) + '.txt', test_pred, fmt='%.8f') - + def calc_correl(self, dev_pred, test_pred): dev_prs, _ = pearsonr(dev_pred, self.dev_y_org) test_prs, _ = pearsonr(test_pred, self.test_y_org) @@ -67,16 +59,14 @@ def evaluate(self, model, epoch, print_info=False): self.dev_loss, self.dev_metric = model.evaluate(self.dev_x, self.dev_y, batch_size=self.batch_size, verbose=0) self.test_loss, self.test_metric = model.evaluate(self.test_x, self.test_y, batch_size=self.batch_size, verbose=0) - # normalized score self.dev_pred = model.predict(self.dev_x, batch_size=self.batch_size).squeeze() self.test_pred = model.predict(self.test_x, batch_size=self.batch_size).squeeze() - # unnormalized score self.dev_pred = self.dataset.convert_to_dataset_friendly_scores(self.dev_pred, self.prompt_id) self.test_pred = self.dataset.convert_to_dataset_friendly_scores(self.test_pred, self.prompt_id) self.dump_predictions(self.dev_pred, self.test_pred, epoch) - + self.dev_prs, self.test_prs, self.dev_spr, self.test_spr, self.dev_tau, self.test_tau = self.calc_correl(self.dev_pred, self.test_pred) self.dev_qwk, self.test_qwk, self.dev_lwk, self.test_lwk = self.calc_qwk(self.dev_pred, self.test_pred) diff --git a/nea/asap_reader.py b/nea/asap_reader.py old mode 100755 new mode 100644 index 4d04a7a..f2a9731 --- a/nea/asap_reader.py +++ b/nea/asap_reader.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import random import codecs import sys @@ -13,9 +7,6 @@ import numpy as np import pickle as pk -reload(sys) -sys.setdefaultencoding('utf8') - logger = logging.getLogger(__name__) num_regex = re.compile('^[+-]?[0-9]+\.?[0-9]*$') ref_scores_dtype = 'int32' @@ -149,35 +140,22 @@ def read_essays(file_path, prompt_id): essays_ids.append(int(tokens[0])) return essays_list, essays_ids -def read_dataset(file_path, output_path, prompt_id, maxlen, vocab, tokenize_text, to_lower, score_index=6, char_level=False): +def read_dataset(file_path, prompt_id, maxlen, vocab, tokenize_text, to_lower, score_index=6, char_level=False): logger.info('Reading dataset from: ' + file_path) - - # modified by sjyan @2017-10-25 - essays = [] - - if 'train' in file_path: - essays_path = output_path + '/preds/train_essays.txt' - if 'dev' in file_path: - essays_path = output_path + '/preds/dev_essays.txt' - if 'test' in file_path: - essays_path = output_path + '/preds/test_essays.txt' - if maxlen > 0: logger.info(' Removing sequences with more than ' + str(maxlen) + ' words') data_x, data_y, prompt_ids = [], [], [] num_hit, unk_hit, total = 0., 0., 0. maxlen_x = -1 with codecs.open(file_path, mode='r', encoding='UTF8') as input_file: - input_file.next() + input_file.next() for line in input_file: tokens = line.strip().split('\t') essay_id = int(tokens[0]) essay_set = int(tokens[1]) - content = tokens[2].strip() # 作文内容 + content = tokens[2].strip() score = float(tokens[score_index]) - if essay_set == prompt_id or prompt_id <= 0: # 如果 作文所属集合编号 等于 命题集编号 - # modified by sjyan @2017-10-25 - essays.append(content) + if essay_set == prompt_id or prompt_id <= 0: if to_lower: content = content.lower() if char_level: @@ -210,17 +188,11 @@ def read_dataset(file_path, output_path, prompt_id, maxlen, vocab, tokenize_text if maxlen_x < len(indices): maxlen_x = len(indices) logger.info(' hit rate: %.2f%%, hit rate: %.2f%%' % (100*num_hit/total, 100*unk_hit/total)) - - # modified by sjyan @2017-10-25 - logger.info('Saving essays to: ' + essays_path) - np.savetxt(essays_path, essays, fmt='%s') - logger.info(' total number of essays: %d' % len(essays)) - return data_x, data_y, prompt_ids, maxlen_x -def get_data(paths, prompt_id, vocab_size, maxlen, output_path, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=None, score_index=6): +def get_data(paths, prompt_id, vocab_size, maxlen, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=None, score_index=6): train_path, dev_path, test_path = paths[0], paths[1], paths[2] - + if not vocab_path: vocab = create_vocab(train_path, prompt_id, maxlen, vocab_size, tokenize_text, to_lower) if len(vocab) < vocab_size: @@ -233,10 +205,9 @@ def get_data(paths, prompt_id, vocab_size, maxlen, output_path, tokenize_text=Tr logger.warning('The vocabualry includes %i words which is different from given: %i' % (len(vocab), vocab_size)) logger.info(' Vocab size: %i' % (len(vocab))) - # modified by sjyan @2017-10-25 - train_x, train_y, train_prompts, train_maxlen = read_dataset(train_path, output_path, prompt_id, maxlen, vocab, tokenize_text, to_lower) - dev_x, dev_y, dev_prompts, dev_maxlen = read_dataset(dev_path, output_path, prompt_id, 0, vocab, tokenize_text, to_lower) - test_x, test_y, test_prompts, test_maxlen = read_dataset(test_path, output_path, prompt_id, 0, vocab, tokenize_text, to_lower) + train_x, train_y, train_prompts, train_maxlen = read_dataset(train_path, prompt_id, maxlen, vocab, tokenize_text, to_lower) + dev_x, dev_y, dev_prompts, dev_maxlen = read_dataset(dev_path, prompt_id, 0, vocab, tokenize_text, to_lower) + test_x, test_y, test_prompts, test_maxlen = read_dataset(test_path, prompt_id, 0, vocab, tokenize_text, to_lower) overal_maxlen = max(train_maxlen, dev_maxlen, test_maxlen) diff --git a/nea/models.py b/nea/models.py old mode 100755 new mode 100644 index d4193e5..af0b405 --- a/nea/models.py +++ b/nea/models.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import numpy as np import logging @@ -76,10 +70,10 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): model.add(Dense(num_outputs)) if not args.skip_init_bias: bias_value = (np.log(initial_mean_value) - np.log(1 - initial_mean_value)).astype(K.floatx()) - model.layers[-1].b.set_value(bias_value) + model.layers[-1].bias = bias_value model.add(Activation('sigmoid')) model.emb_index = 0 - + elif args.model_type == 'breg': logger.info('Building a BIDIRECTIONAL REGRESSION model') from keras.layers import Dense, Dropout, Embedding, LSTM, Input, merge @@ -136,7 +130,8 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing lookup table') emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - model.layers[model.emb_index].W.set_value(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].W.get_value())) + # modified by Shengjia Yan @2017-11-06 Monday + model.layers[model.emb_index].set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].get_weights())) logger.info(' Done') return model diff --git a/nea/my_kappa_calculator.py b/nea/my_kappa_calculator.py old mode 100755 new mode 100644 index 4f68cc3..dd7310b --- a/nea/my_kappa_calculator.py +++ b/nea/my_kappa_calculator.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import numpy as np from quadratic_weighted_kappa import quadratic_weighted_kappa as qwk from quadratic_weighted_kappa import linear_weighted_kappa as lwk diff --git a/nea/my_layers.py b/nea/my_layers.py old mode 100755 new mode 100644 index 91707e9..c413b2b --- a/nea/my_layers.py +++ b/nea/my_layers.py @@ -1,14 +1,9 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import keras.backend as K from keras.engine.topology import Layer from keras.layers.convolutional import Convolution1D import numpy as np import sys +# import tensorflow as tf class Attention(Layer): def __init__(self, op='attsum', activation='tanh', init_stdev=0.01, **kwargs): @@ -60,11 +55,12 @@ def __init__(self, mask_zero=True, **kwargs): def call(self, x, mask=None): if self.mask_zero: - return K.cast(x.sum(axis=1) / mask.sum(axis=1, keepdims=True), K.floatx()) + mask = K.cast(mask, K.floatx()) + return K.cast(K.sum(x, axis=1) / K.sum(mask, axis=1, keepdims= True), K.floatx()) else: return K.mean(x, axis=1) - def get_output_shape_for(self, input_shape): + def compute_output_shape(self, input_shape): return (input_shape[0], input_shape[2]) def compute_mask(self, x, mask): @@ -79,6 +75,7 @@ class Conv1DWithMasking(Convolution1D): def __init__(self, **kwargs): self.supports_masking = True super(Conv1DWithMasking, self).__init__(**kwargs) + def compute_mask(self, x, mask): return mask diff --git a/nea/optimizers.py b/nea/optimizers.py old mode 100755 new mode 100644 index 099fcce..556eb53 --- a/nea/optimizers.py +++ b/nea/optimizers.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import keras.optimizers as opt def get_optimizer(args): diff --git a/nea/plot_confusion_matrix.py b/nea/plot_confusion_matrix.py deleted file mode 100644 index 5531deb..0000000 --- a/nea/plot_confusion_matrix.py +++ /dev/null @@ -1,98 +0,0 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-26 -# Email: i@yanshengjia.com - -import sys -reload(sys) -sys.setdefaultencoding('utf8') - -import logging -import numpy as np -import itertools -import matplotlib.pyplot as plt -from sklearn.metrics import confusion_matrix - - -def load_confusion_matrix(ref_score, pred_score): - ref_file = open('../output/emb/rnn/prompt_1/fold_0/preds/dev_ref.txt', 'r') - pred_file = open('../output/emb/rnn/prompt_1/fold_0/preds/dev_pred_49.txt', 'r') - - for ref in ref_file.readlines(): - ref = ref.strip('\n') - ref = int(ref) - ref_score.append(ref) - for pred in pred_file.readlines(): - pred = pred.strip('\n') - pred = float(pred) - pred = round(pred) - pred_score.append(pred) - -def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues): - """ - This function prints and plots the confusion matrix. - Normalization can be applied by setting `normalize=True`. - """ - if normalize: - # 1. find out how many samples per class have received their correct label - # 计算真正类别为k的样本被预测成各个类别的比例 - # e.g. 有25个样本的 true label 是 6，其中10个样本被预测为类别7，那么在混淆矩阵中 true label = 6 并且 predicted label = 7 的一个格子中的值为 0.4 - cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] - - # 2. get the precision (fraction of class-k predictions that have ground truth label k) - # 计算预测的准确率 - # e.g. 预测为类别k的有12个，但其中只有9个的真正类别是k，那么准确率为 0.75 - # cm = cm.astype('float') / cm.sum(axis=0)[:, np.newaxis] - - print("Normalized confusion matrix") - else: - print('Confusion matrix, without normalization') - - print(cm) - - plt.imshow(cm, interpolation='nearest', cmap=cmap) - plt.title(title) - plt.colorbar() - tick_marks = np.arange(len(classes)) - plt.xticks(tick_marks, classes, rotation=45) - plt.yticks(tick_marks, classes) - - fmt = '.2f' if normalize else 'd' - thresh = cm.max() / 2. - for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): - plt.text(j, i, format(cm[i, j], fmt), - horizontalalignment="center", - color="white" if cm[i, j] > thresh else "black") - - # plt.tight_layout() - plt.ylabel('True label') - plt.xlabel('Predicted label') - -def main(): - ref_score = [] # true label - pred_score = [] # predicted label - - load_confusion_matrix(ref_score, pred_score) - - nea_matrix = confusion_matrix(ref_score, pred_score) - np.set_printoptions(precision=2) - class_names = ['2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12'] - - # Plot non-normalized confusion matrix - plt.figure() - plot_confusion_matrix(nea_matrix, classes=class_names, title='Confusion matrix, without normalization') - plt.savefig('./unnormalized_cm.png') - - # Plot normalized confusion matrix - plt.figure() - plot_confusion_matrix(nea_matrix, classes=class_names, normalize=True, title='Normalized confusion matrix') - plt.savefig('./normalized_cm.png') - - # plt.show() - -if __name__ == '__main__': - main() - - - diff --git a/nea/quadratic_weighted_kappa.py b/nea/quadratic_weighted_kappa.py old mode 100755 new mode 100644 index 23d5184..7609840 --- a/nea/quadratic_weighted_kappa.py +++ b/nea/quadratic_weighted_kappa.py @@ -1,8 +1,4 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com +#! /usr/bin/env python2.7 import numpy as np diff --git a/nea/utils.py b/nea/utils.py old mode 100755 new mode 100644 index 37ef24f..6201f1c --- a/nea/utils.py +++ b/nea/utils.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import sys import os, errno import logging diff --git a/nea/w2vEmbReader.py b/nea/w2vEmbReader.py old mode 100755 new mode 100644 index 5007c94..530d6f4 --- a/nea/w2vEmbReader.py +++ b/nea/w2vEmbReader.py @@ -1,9 +1,3 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com - import codecs import logging import numpy as np @@ -34,12 +28,10 @@ def __init__(self, emb_path, emb_dim=None): counter = 0 for line in emb_file: tokens = line.split() - # modified by sjyan @2017-10-19 - # deal with comma + # modified by Shengjia Yan @2017-11-03 Friday word = tokens[0] - str = tokens[1] - vec = str.split(',') - assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' + vec = tokens[1].split(',') + assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' self.embeddings[word] = vec counter += 1 assert counter == self.vocab_size, 'Vocab size does not match the header info' @@ -50,18 +42,15 @@ def __init__(self, emb_path, emb_dim=None): self.embeddings = {} for line in emb_file: tokens = line.split() - # modified by sjyan @2017-10-19 - # deal with comma + # modified by Shengjia Yan @2017-11-03 Friday word = tokens[0] - str = tokens[1] - vec = str.split(',') - + vec = tokens[1].split(',') if self.emb_dim == -1: self.emb_dim = len(vec) assert self.emb_dim == emb_dim, 'The embeddings dimension does not match with the requested dimension' else: assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' - + self.embeddings[word] = vec self.vocab_size += 1 @@ -77,7 +66,8 @@ def get_emb_matrix_given_vocab(self, vocab, emb_matrix): counter = 0. for word, index in vocab.iteritems(): try: - emb_matrix[index] = self.embeddings[word] + # modified by Shengjia Yan @2017-11-06 Monday + emb_matrix[0][index] = self.embeddings[word] counter += 1 except KeyError: pass diff --git a/train_nea.py b/train_nea.py index 9af3423..5a6f995 100755 --- a/train_nea.py +++ b/train_nea.py @@ -1,8 +1,4 @@ -# !/usr/bin/python -# -*- coding:utf-8 -*- -# Author: Shengjia Yan -# Date: 2017-10-19 -# Email: i@yanshengjia.com +#!/usr/bin/env python import os import argparse @@ -14,6 +10,7 @@ import nea.utils as U import pickle as pk + os.environ['KERAS_BACKEND']='theano' logger = logging.getLogger(__name__) @@ -32,7 +29,7 @@ parser.add_argument("-a", "--algorithm", dest="algorithm", type=str, metavar='', default='rmsprop', help="Optimization algorithm (rmsprop|sgd|adagrad|adadelta|adam|adamax) (default=rmsprop)") parser.add_argument("-l", "--loss", dest="loss", type=str, metavar='', default='mse', help="Loss function (mse|mae) (default=mse)") parser.add_argument("-e", "--embdim", dest="emb_dim", type=int, metavar='', default=50, help="Embeddings dimension (default=50)") -parser.add_argument("-c", "--cnndim", dest="cnn_dim", type=int, metavar='', default=10, help="CNN output dimension. '0' means no CNN layer (default=0)") +parser.add_argument("-c", "--cnndim", dest="cnn_dim", type=int, metavar='', default=0, help="CNN output dimension. '0' means no CNN layer (default=0)") parser.add_argument("-w", "--cnnwin", dest="cnn_window_size", type=int, metavar='', default=3, help="CNN window size. (default=3)") parser.add_argument("-r", "--rnndim", dest="rnn_dim", type=int, metavar='', default=300, help="RNN dimension. '0' means no RNN layer (default=300)") parser.add_argument("-b", "--batch-size", dest="batch_size", type=int, metavar='', default=32, help="Batch size (default=32)") @@ -75,9 +72,8 @@ from keras.preprocessing import sequence # data_x is a list of lists -# modified by sjyan @2017-10-25 (train_x, train_y, train_pmt), (dev_x, dev_y, dev_pmt), (test_x, test_y, test_pmt), vocab, vocab_size, overal_maxlen, num_outputs = dataset.get_data( - (args.train_path, args.dev_path, args.test_path), args.prompt_id, args.vocab_size, args.maxlen, args.out_dir_path, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=args.vocab_path) + (args.train_path, args.dev_path, args.test_path), args.prompt_id, args.vocab_size, args.maxlen, tokenize_text=True, to_lower=True, sort_by_len=False, vocab_path=args.vocab_path) # Dump vocab with open(out_dir + '/vocab.pkl', 'wb') as vocab_file: @@ -135,7 +131,6 @@ logger.info(' train_y mean: %s, stdev: %s, MFC: %s' % (str(train_mean), str(train_std), str(mfs_list))) # We need the dev and test sets in the original scale for evaluation -train_y_org = train_y.astype(dataset.get_ref_dtype()) dev_y_org = dev_y.astype(dataset.get_ref_dtype()) test_y_org = test_y.astype(dataset.get_ref_dtype()) @@ -172,9 +167,9 @@ ## Plotting model # -from keras.utils.visualize_util import plot +#from keras.utils.visualize_util import plot -plot(model, to_file = out_dir + '/model.png') +#plot(model, to_file = out_dir + '/model.png') ############################################################################################################################### ## Save model architecture @@ -184,12 +179,12 @@ with open(out_dir + '/model_arch.json', 'w') as arch: arch.write(model.to_json(indent=2)) logger.info(' Done') - + ############################################################################################################################### ## Evaluator # -evl = Evaluator(dataset, args.prompt_id, out_dir, dev_x, test_x, dev_y, test_y, train_y_org, dev_y_org, test_y_org) +evl = Evaluator(dataset, args.prompt_id, out_dir, dev_x, test_x, dev_y, test_y, dev_y_org, test_y_org) ############################################################################################################################### ## Training @@ -197,14 +192,16 @@ logger.info('--------------------------------------------------------------------------------------------------------------------------') logger.info('Initial Evaluation:') -evl.evaluate(model, -1, print_info=True) +#evl.evaluate(model, -1, print_info=True) total_train_time = 0 total_eval_time = 0 + for ii in range(args.epochs): # Training t0 = time() + train_history = model.fit(train_x, train_y, batch_size=args.batch_size, nb_epoch=1, verbose=0) tr_time = time() - t0 total_train_time += tr_time From 9e45b3bd828d59840d821ce7f2650b59ec1e4776 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 15:45:33 +0800 Subject: [PATCH 05/11] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f8d2524..3e96708 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,8 @@ An automatic essay scoring system based on convolutional and recurrent neural ne - Keras 2.0.8 - Theano 0.9.0 -- Tensorflow 1.3.0 +- Numpy 1.13.3 +- Scipy 1.0.0 - Python 2.7.9 ### Data ### @@ -54,6 +55,7 @@ Neural Essay Assessor is licensed under the GNU General Public License Version 3 * Kaveh Taghipour (kaveh@comp.nus.edu.sg) * Hwee Tou Ng (nght@comp.nus.edu.sg) +* Shengjia Yan (i@yanshengjia.com) ### Publication ### From 3ef9efb509bc056509959b8479fc57481190d73a Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 17:14:21 +0800 Subject: [PATCH 06/11] Clean the repo --- README.md | 11 +---------- nea/models.py | 1 - nea/my_layers.py | 2 -- nea/w2vEmbReader.py | 3 --- train_nea.py | 7 +++---- 5 files changed, 4 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3e96708..b11739f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Neural Essay Assessor 2.0 # +# Neural Essay Assessor # An automatic essay scoring system based on convolutional and recurrent neural networks, including GRU and LSTM. @@ -8,14 +8,6 @@ An automatic essay scoring system based on convolutional and recurrent neural ne * Prepare data * Run train_nea.py -### Environment - -- Keras 2.0.8 -- Theano 0.9.0 -- Numpy 1.13.3 -- Scipy 1.0.0 -- Python 2.7.9 - ### Data ### We have used 5-fold cross validation on ASAP dataset to evaluate our system. This dataset (training_set_rel3.tsv) can be downloaded from [here](https://www.kaggle.com/c/asap-aes/data). After downloading the file, put it in the [data](https://github.com/nusnlp/nea/tree/master/data) directory and create training, development and test data using ```preprocess_asap.py``` script: @@ -55,7 +47,6 @@ Neural Essay Assessor is licensed under the GNU General Public License Version 3 * Kaveh Taghipour (kaveh@comp.nus.edu.sg) * Hwee Tou Ng (nght@comp.nus.edu.sg) -* Shengjia Yan (i@yanshengjia.com) ### Publication ### diff --git a/nea/models.py b/nea/models.py index af0b405..3db0cb8 100644 --- a/nea/models.py +++ b/nea/models.py @@ -130,7 +130,6 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing lookup table') emb_reader = EmbReader(args.emb_path, emb_dim=args.emb_dim) - # modified by Shengjia Yan @2017-11-06 Monday model.layers[model.emb_index].set_weights(emb_reader.get_emb_matrix_given_vocab(vocab, model.layers[model.emb_index].get_weights())) logger.info(' Done') diff --git a/nea/my_layers.py b/nea/my_layers.py index c413b2b..78137ff 100644 --- a/nea/my_layers.py +++ b/nea/my_layers.py @@ -3,7 +3,6 @@ from keras.layers.convolutional import Convolution1D import numpy as np import sys -# import tensorflow as tf class Attention(Layer): def __init__(self, op='attsum', activation='tanh', init_stdev=0.01, **kwargs): @@ -76,6 +75,5 @@ def __init__(self, **kwargs): self.supports_masking = True super(Conv1DWithMasking, self).__init__(**kwargs) - def compute_mask(self, x, mask): return mask diff --git a/nea/w2vEmbReader.py b/nea/w2vEmbReader.py index 530d6f4..45d9367 100644 --- a/nea/w2vEmbReader.py +++ b/nea/w2vEmbReader.py @@ -28,7 +28,6 @@ def __init__(self, emb_path, emb_dim=None): counter = 0 for line in emb_file: tokens = line.split() - # modified by Shengjia Yan @2017-11-03 Friday word = tokens[0] vec = tokens[1].split(',') assert len(vec) == self.emb_dim, 'The number of dimensions does not match the header info' @@ -42,7 +41,6 @@ def __init__(self, emb_path, emb_dim=None): self.embeddings = {} for line in emb_file: tokens = line.split() - # modified by Shengjia Yan @2017-11-03 Friday word = tokens[0] vec = tokens[1].split(',') if self.emb_dim == -1: @@ -66,7 +64,6 @@ def get_emb_matrix_given_vocab(self, vocab, emb_matrix): counter = 0. for word, index in vocab.iteritems(): try: - # modified by Shengjia Yan @2017-11-06 Monday emb_matrix[0][index] = self.embeddings[word] counter += 1 except KeyError: diff --git a/train_nea.py b/train_nea.py index 5a6f995..45c9547 100755 --- a/train_nea.py +++ b/train_nea.py @@ -167,9 +167,9 @@ ## Plotting model # -#from keras.utils.visualize_util import plot +from keras.utils.visualize_util import plot -#plot(model, to_file = out_dir + '/model.png') +plot(model, to_file = out_dir + '/model.png') ############################################################################################################################### ## Save model architecture @@ -192,7 +192,7 @@ logger.info('--------------------------------------------------------------------------------------------------------------------------') logger.info('Initial Evaluation:') -#evl.evaluate(model, -1, print_info=True) +evl.evaluate(model, -1, print_info=True) total_train_time = 0 total_eval_time = 0 @@ -201,7 +201,6 @@ for ii in range(args.epochs): # Training t0 = time() - train_history = model.fit(train_x, train_y, batch_size=args.batch_size, nb_epoch=1, verbose=0) tr_time = time() - t0 total_train_time += tr_time From c54034762dfcd78070a97f64b5cf5ee1c4cdca7f Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 17:26:55 +0800 Subject: [PATCH 07/11] Fix a bug related to plotting --- train_nea.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train_nea.py b/train_nea.py index 45c9547..6ddb78c 100755 --- a/train_nea.py +++ b/train_nea.py @@ -167,9 +167,9 @@ ## Plotting model # -from keras.utils.visualize_util import plot +from keras.utils.vis_utils import plot_model -plot(model, to_file = out_dir + '/model.png') +plot_model(model, to_file = out_dir + '/model.png') ############################################################################################################################### ## Save model architecture From 754f1c6d5377b89b5c175357447d47a9fead3d47 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 8 Nov 2017 18:30:51 +0800 Subject: [PATCH 08/11] Fix a bug related to bias --- nea/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nea/models.py b/nea/models.py index 3db0cb8..f6e2b34 100644 --- a/nea/models.py +++ b/nea/models.py @@ -49,7 +49,7 @@ def create_model(args, initial_mean_value, overal_maxlen, vocab): model.add(Dense(num_outputs)) if not args.skip_init_bias: bias_value = (np.log(initial_mean_value) - np.log(1 - initial_mean_value)).astype(K.floatx()) - model.layers[-1].b.set_value(bias_value) + model.layers[-1].bias = bias_value model.add(Activation('sigmoid')) model.emb_index = 0 From 0ec7fc2add8d087e94bd6945fd5395cba30fb697 Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 17 Jan 2018 14:17:32 +0800 Subject: [PATCH 09/11] Update README --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index b11739f..e1bd352 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,16 @@ An automatic essay scoring system based on convolutional and recurrent neural networks, including GRU and LSTM. +Compatible with Keras 2. + +## Environment + +* Keras 2.0.8 +* Theano 0.9.0 +* Numpy 1.13.3 +* Scipy 1.0.0 +* Python 2.7.9 + ### Set Up ### * Install Keras (with Theano backend) From cdabfeebafb87fe364a90d0dc0d3771af87d6edf Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 17 Jan 2018 14:21:27 +0800 Subject: [PATCH 10/11] Update README --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index e1bd352..0412585 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ An automatic essay scoring system based on convolutional and recurrent neural networks, including GRU and LSTM. -Compatible with Keras 2. - ## Environment * Keras 2.0.8 From a6ffad8e1f266304b1da62ea7e2c618d25fde27d Mon Sep 17 00:00:00 2001 From: sjyan Date: Wed, 17 Jan 2018 14:46:40 +0800 Subject: [PATCH 11/11] Rename nb_epoch to epochs --- train_nea.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_nea.py b/train_nea.py index 6ddb78c..7b544e6 100755 --- a/train_nea.py +++ b/train_nea.py @@ -201,7 +201,7 @@ for ii in range(args.epochs): # Training t0 = time() - train_history = model.fit(train_x, train_y, batch_size=args.batch_size, nb_epoch=1, verbose=0) + train_history = model.fit(train_x, train_y, batch_size=args.batch_size, epochs=1, verbose=0) tr_time = time() - t0 total_train_time += tr_time