linguistic_multitask_learning/jPTDP.py at master · andy-yangz/linguistic_multitask_learning · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
# coding=utf-8
from optparse import OptionParser
import pickle, utils, learner, os, os.path, time
from collections import defaultdict
import os.path
from random import randint


if __name__ == '__main__':
    parser = OptionParser()
    parser.add_option("--train", dest="conll_train", help="Path to annotated CONLL train file", metavar="FILE", default="N/A")
    parser.add_option("--dev", dest="conll_dev", help="Path to annotated CONLL dev file", metavar="FILE", default="N/A")
    parser.add_option("--test", dest="conll_test", help="Path to CONLL test file", metavar="FILE", default="N/A")
    parser.add_option("--gold", dest="conll_gold", help="Path to CONLL test gold file", metavar="FILE", default="N/A")
    parser.add_option("--output", dest="conll_test_output", help="File name for predicted output", metavar="FILE", default="N/A")
    parser.add_option("--extrn", dest="external_embedding", help="External embeddings", metavar="FILE")
    parser.add_option("--pre_wembed", dest="pretrain_wembed", help="Pretrained Word embeddings", metavar="FILE")
    parser.add_option("--params", dest="params", help="Parameters file", metavar="FILE", default="model.params")
    parser.add_option("--model", dest="model", help="Load/Save model file", metavar="FILE", default="model")
    parser.add_option("--wembedding", type="int", dest="wembedding_dims", default=100)
    parser.add_option("--cembedding", type="int", dest="cembedding_dims", default=64)
    parser.add_option("--membedding", type="int", dest="membedding_dims", default=64)
    parser.add_option("--pembedding", type="int", dest="pembedding_dims", default=32)
    parser.add_option("--pos_layer", type="int", dest="pos_layer", default=1)
    parser.add_option("--dep_layer", type="int", dest="dep_layer", default=2)
    parser.add_option("--pos_dropout", type="float", dest="pos_dropout", default=0.2)
    parser.add_option("--dep_dropout", type="float", dest="dep_dropout", default=0.2)
    parser.add_option("--epochs", type="int", dest="epochs", default=30)
    parser.add_option("--arc_hidden", type="int", dest="arc_hidden", default=100)
    parser.add_option("--rel_hidden", type="int", dest="rel_hidden", default=100)
    parser.add_option("--hidden2", type="int", dest="hidden2_units", default=0)
    # parser.add_option("--lr", type="float", dest="learning_rate", default=0.001)
    parser.add_option("--outdir", type="string", dest="outdir", default="results")
    parser.add_option("--activation", type="string", dest="activation", default="tanh")
    parser.add_option("--rnn_type", type="string", dest="rnn_type", default="LSTM")
    parser.add_option("--lstmlayers", type="int", dest="lstm_layers", default=2)
    parser.add_option("--pos_lstm_dims", type="int", dest="pos_lstm_dims", default=128)
    parser.add_option("--dep_lstm_dims", type="int", dest="dep_lstm_dims", default=128)
    parser.add_option("--gold_pos", action="store_true", dest="gold_pos", default=False)
    parser.add_option("--disableblstm", action="store_false", dest="blstmFlag", default=True)
    parser.add_option("--disablelabels", action="store_false", dest="labelsFlag", default=True)
    parser.add_option("--predict", action="store_true", dest="predictFlag", default=False)
    parser.add_option("--error_ana", action="store_true", dest="error_ana", default=False)
    parser.add_option("--disablecostaug", action="store_false", dest="costaugFlag", default=True)
    parser.add_option("--dynet-seed", type="int", dest="seed", default=randint(0, 1e8))
    parser.add_option("--dynet-mem", type="int", dest="mem", default=0)
    parser.add_option("--exp_times", type="int", dest='exp_times', default=0)
    parser.add_option("--log", dest="log_path", help="Path to log file", metavar="FILE", default="./log")

    (options, args) = parser.parse_args()

    print 'Using external embedding:', options.external_embedding

    if options.predictFlag:
        with open(os.path.join(options.outdir, options.params), 'r') as paramsfp:
            words, w2i, c2i, pos, rels, morphs, stored_opt = pickle.load(paramsfp)

        stored_opt.external_embedding = options.external_embedding
        stored_opt.pretrain_wembed = options.pretrain_wembed

        print 'Loading pre-trained joint model'
        parser = learner.jPosDepLearner(words, pos, rels, morphs, w2i, c2i, stored_opt)
        parser.Load(os.path.join(options.outdir, os.path.basename(options.model)))
        conllu = (os.path.splitext(options.conll_test.lower())[1] == '.conllu')
        tespath = os.path.join(options.outdir, stored_opt.model + 'test_pred.conllu')
        print 'Predicting POS tags and parsing dependencies'
        devPredSents = parser.Predict(options.conll_test)

        te = time.time()
        print 'Finished in', te-ts, 'seconds.'
        utils.write_conll(tespath, test_res)

        if not conllu:#Scored with punctuation
           os.system('perl utils/eval07.pl -q -g ' + options.conll_test + ' -s ' + tespath  + ' > ' + tespath + '.scores.txt')
        else:
           os.system('python utils/evaluation_script/conll17_ud_eval.py -v -w utils/evaluation_script/weights.clas ' + options.conll_gold + ' ' + tespath + ' > ' + tespath + '.scores.txt')
    else:
        if os.path.isfile(os.path.join(options.outdir, options.params)):
            print("Load existed vocabulary.")
            with open(os.path.join(options.outdir, options.params), 'r') as paramsfp:
                words, w2i, c2i, pos, rels, morphs, stored_opt = pickle.load(paramsfp)
        else:
            print 'Extracting vocabulary'
            words, w2i, c2i, pos, rels, morphs = utils.vocab(options.conll_train)

            with open(os.path.join(options.outdir, options.params), 'w') as paramsfp:
                pickle.dump((words, w2i, c2i, pos, rels, morphs, options), paramsfp)

        print 'Initializing joint model'
        print 'RNN type: ' + options.rnn_type
        print 'POS layer: %d, POS LSTM dims: %d' % (options.pos_layer, options.pos_lstm_dims)
        print 'Dep layer: %d, Dep LSTM dims: %d' % (options.dep_layer, options.dep_lstm_dims)
        parser = learner.jPosDepLearner(words, pos, rels, morphs, w2i, c2i, options)

        highestScore = 0.0
        eId = 0

        for epoch in xrange(options.epochs):
            print '\n-----------------\nStarting epoch', epoch + 1
            parser.Train(options.conll_train)

            if options.conll_dev == "N/A":
                parser.Save(os.path.join(options.outdir, os.path.basename(options.model)))

            else:
                devPredSents = parser.Predict(options.conll_dev)

                count = 0
                uasCount = 0
                lasCount = 0
                posCount = 0
                morphCount = 0
                poslasCount = 0
                for idSent, devSent in enumerate(devPredSents):
                    conll_devSent = [entry for entry in devSent if isinstance(entry, utils.ConllEntry)]

                    for entry in conll_devSent:
                        if entry.id <= 0:
                            continue
                        if entry.pos == entry.pred_pos and entry.parent_id == entry.pred_parent_id and entry.pred_relation == entry.relation:
                            poslasCount += 1
                        if entry.pos == entry.pred_pos:
                            posCount += 1
                        if entry.feats == entry.pred_feats:
                            morphCount += 1
                        if entry.parent_id == entry.pred_parent_id:
                            uasCount += 1
                        if entry.parent_id == entry.pred_parent_id and entry.pred_relation == entry.relation:
                            lasCount += 1
                        count += 1

                LAS = float(lasCount) * 100 / count
                UAS = float(uasCount) * 100 / count
                POS = float(posCount) * 100 / count
                Morph = float(morphCount) * 100 / count
                POSLAS = float(poslasCount) * 100 / count

                print "---\nLAS accuracy:\t%.2f" % LAS
                print "UAS accuracy:\t%.2f" % UAS
                print "POS accuracy:\t%.2f" % POS
                print "Morph accuracy:\t%.2f" % Morph
                print "POS&LAS:\t%.2f" % POSLAS

                score = float(poslasCount) * 100 / count
                if score >= highestScore:
                    parser.Save(os.path.join(options.outdir, os.path.basename(options.model)))
                    highestScore = score
                    eId = epoch + 1

        print "Highest POS&LAS: %.2f at epoch %d" % (highestScore, eId)
        with open(os.path.join(options.log_path, os.path.basename(options.model)+'.log.dev'), 'a') as log_f:
            log_f.write('%d times of experiments.\n' % options.exp_times)
            log_f.write('LAS\tUAS\tPOS\tMorph\tPOS&LAS\tEpoch\n')
            log_f.write('%.2f\t%.2f\t%.2f\t%.2f\t%.2f\t%d\n'%(LAS, UAS, POS, Morph, POSLAS, eId))