diff --git a/CHANGES.md b/CHANGES.md index 0dd29d1da..01500ebab 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,9 @@ # Changes ++ **8.3.0** - Added support for proper multi-threaded execution of CluProcessor by cloning parameters in each thread. + **8.2.3** - Do not allow loops in enhanced semantic roles. + **8.2.2** - Bug fix: we now guarantee that the SRL graph has the same number of nodes as the words in the sentence. ++ **8.2.2** - Bug fixes in the generation of enhanced semantic roles. ++ **8.2.1** - Improvements in the generation of enhanced semantic roles. + **8.2.1** - Bug fix in LexiconNER: we were ignoring case information before. + **8.2.1** - Improvements to the generation of enhanced semantic roles. + **8.2.0** - Added simple lexicon-based sentiment analyzer, using Bing Liu's lexicon. diff --git a/corenlp/src/main/scala/org/clulab/processors/ProcessorShell.scala b/corenlp/src/main/scala/org/clulab/processors/ProcessorShell.scala index a0abfecc3..d07fc351b 100644 --- a/corenlp/src/main/scala/org/clulab/processors/ProcessorShell.scala +++ b/corenlp/src/main/scala/org/clulab/processors/ProcessorShell.scala @@ -67,7 +67,7 @@ object ProcessorShell extends App { case ":clu" => reader.setPrompt("(clu)>>> ") println("Preparing CluProcessor...\n") - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) proc = clu proc.annotate("initialize me!") diff --git a/corenlp/src/main/scala/org/clulab/processors/examples/ParallelProcessorExample.scala b/corenlp/src/main/scala/org/clulab/processors/examples/ParallelProcessorExample.scala index bb547db78..94a4acb85 100644 --- a/corenlp/src/main/scala/org/clulab/processors/examples/ParallelProcessorExample.scala +++ b/corenlp/src/main/scala/org/clulab/processors/examples/ParallelProcessorExample.scala @@ -13,6 +13,8 @@ import org.clulab.dynet.Utils import org.clulab.processors.Document import org.clulab.processors.Processor import org.clulab.processors.clu.CluProcessor +import org.clulab.processors.fastnlp.FastNLPProcessor +import org.clulab.processors.fastnlp.FastNLPProcessorWithSemanticRoles import org.clulab.serialization.DocumentSerializer import scala.collection.parallel.ForkJoinTaskSupport @@ -96,7 +98,7 @@ object ParallelProcessorExample { forkJoinPoolConstructor.newInstance(threads.asInstanceOf[Integer]) // For the record, this is the standard version - //new ForkJoinPool(threads) + // new ForkJoinPool(threads) } val forkJoinPool = newForkJoinPool(threads) @@ -113,22 +115,33 @@ object ParallelProcessorExample { val outputDir = args(1) val extension = args(2) val threads = args(3).toInt + val parallel = true val files = findFiles(inputDir, extension) + val sortedFiles = files.sortBy(-_.length) // Parallelizing by file results in a quick crash. - val parFiles = parallelize(files, threads) + val parFiles = parallelize(sortedFiles, threads) - Utils.initializeDyNet() + val startupTimer = new Timer("This is how long it takes to start up") + startupTimer.start() + + Utils.initializeDyNet(train = !parallel) val processor: Processor = new CluProcessor() +// val processor: Processor = new FastNLPProcessor() +// val processor: Processor = new FastNLPProcessorWithSemanticRoles() + val documentSerializer = new DocumentSerializer val untimed = processor.annotate("I am happy to join with you today in what will go down in history as the greatest demonstration for freedom in the history of our nation.") + startupTimer.stop() + println(startupTimer.toString) + val timer = new Timer(s"$threads threads processing ${parFiles.size} files") timer.start() - parFiles.foreach { file => + (if (parallel) parFiles else files).foreach { file => println(s"Processing ${file.getName}...") val text = { @@ -140,7 +153,15 @@ object ParallelProcessorExample { } val outputFile = new File(outputDir + "/" + file.getName) - val document = processor.annotate(text) + val document = try { + val document = processor.annotate(text) + document + } + catch { + case throwable: Throwable => + println(s"Threw exception for ${file.getName}") + throw throwable + } val printedDocument = { val stringWriter = new StringWriter val printWriter = new PrintWriter(stringWriter) diff --git a/corenlp/src/main/scala/org/clulab/processors/fastnlp/FastNLPProcessorWithSemanticRoles.scala b/corenlp/src/main/scala/org/clulab/processors/fastnlp/FastNLPProcessorWithSemanticRoles.scala index 73bb02173..aac1581c8 100644 --- a/corenlp/src/main/scala/org/clulab/processors/fastnlp/FastNLPProcessorWithSemanticRoles.scala +++ b/corenlp/src/main/scala/org/clulab/processors/fastnlp/FastNLPProcessorWithSemanticRoles.scala @@ -17,7 +17,7 @@ class FastNLPProcessorWithSemanticRoles(tokenizerPostProcessor:Option[TokenizerS /** Used for SRL */ lazy val cluProcessor = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) new CluProcessor() } diff --git a/main/build.sbt b/main/build.sbt index f993d686a..3e97083b4 100644 --- a/main/build.sbt +++ b/main/build.sbt @@ -27,7 +27,7 @@ libraryDependencies ++= { // for machine learning "de.bwaldvogel" % "liblinear" % "2.30", "tw.edu.ntu.csie" % "libsvm" % "3.23", - "org.clulab" %% "fatdynet" % "0.2.5", // "0-cuda.2.6-SNAPSHOT", // "0.2.5" + "org.clulab" %% "fatdynet" % "0.2.6", // "0-cuda.2.6-SNAPSHOT", // "0.2.5" // NLP tools used by CluProcessor "org.antlr" % "antlr4-runtime" % "4.6", // for tokenization diff --git a/main/src/main/scala/org/clulab/dynet/CnnExample.scala b/main/src/main/scala/org/clulab/dynet/CnnExample.scala index 4e88cdc32..f374fa217 100644 --- a/main/src/main/scala/org/clulab/dynet/CnnExample.scala +++ b/main/src/main/scala/org/clulab/dynet/CnnExample.scala @@ -8,7 +8,7 @@ import edu.cmu.dynet.{Dim, Expression, ParameterCollection, UnsignedVector} // https://github.com/neubig/nn4nlp-code/blob/970d91a51664b3d91a9822b61cd76abea20218cb/05-cnn/cnn-class.py#L45 // object CnnExample extends App { - Utils.initializeDyNet() + Utils.initializeDyNet(train = true) val pc = new ParameterCollection() val embSize = 3 diff --git a/main/src/main/scala/org/clulab/dynet/CoNLLSRLToMetal.scala b/main/src/main/scala/org/clulab/dynet/CoNLLSRLToMetal.scala index f7202e372..bd9276258 100644 --- a/main/src/main/scala/org/clulab/dynet/CoNLLSRLToMetal.scala +++ b/main/src/main/scala/org/clulab/dynet/CoNLLSRLToMetal.scala @@ -363,7 +363,7 @@ object CoNLLSRLToMetal { def main(args: Array[String]): Unit = { assert(args.length == 2) - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) val file = new File(args(0)) val reader = new CoNLLSRLToMetal diff --git a/main/src/main/scala/org/clulab/dynet/ConstEmbeddingsGlove.scala b/main/src/main/scala/org/clulab/dynet/ConstEmbeddingsGlove.scala index 198d40149..1e02b4c5f 100644 --- a/main/src/main/scala/org/clulab/dynet/ConstEmbeddingsGlove.scala +++ b/main/src/main/scala/org/clulab/dynet/ConstEmbeddingsGlove.scala @@ -34,6 +34,7 @@ class ConstEmbeddingsGlove(matrixResourceName: String, isResource:Boolean = true override def dim: Int = lookupParameters.dim().get(0).toInt + /** Read-only access to the embedding for this word */ def get(word:String): Expression = { if(w2i.contains(word)) { Expression.constLookup(lookupParameters, w2i(word)) @@ -70,7 +71,7 @@ object ConstEmbeddingsGlove { def apply(matrixResourceName: String, isResource: Boolean): ConstEmbeddingsGlove = { - DyNetSync.synchronized { + this.synchronized { // these objects are read-only and they use a lot of RAM, so let's reuse them if they exist if(SINGLETON.isEmpty) { SINGLETON = Some(new ConstEmbeddingsGlove(matrixResourceName, isResource)) diff --git a/main/src/main/scala/org/clulab/dynet/DyNetSync.scala b/main/src/main/scala/org/clulab/dynet/DyNetSync.scala deleted file mode 100644 index fb06ea107..000000000 --- a/main/src/main/scala/org/clulab/dynet/DyNetSync.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.clulab.dynet - -/** Use this object for synchronized statements in DyNet */ -object DyNetSync { - -} diff --git a/main/src/main/scala/org/clulab/dynet/EmbeddingLayer.scala b/main/src/main/scala/org/clulab/dynet/EmbeddingLayer.scala index ca12f3443..3abc25923 100644 --- a/main/src/main/scala/org/clulab/dynet/EmbeddingLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/EmbeddingLayer.scala @@ -17,33 +17,40 @@ import scala.util.Random * This layer takes a sequence of words and produces a sequence of Expression that stores the words' full embeddings * @author Mihai */ -class EmbeddingLayer (val parameters:ParameterCollection, - val w2i:Map[String, Int], // word to index - val w2f:Counter[String], // word to frequency - val c2i:Map[Char, Int], // character to index - val tag2i:Option[Map[String, Int]], // POS tag to index - val ne2i:Option[Map[String, Int]], // NE tag to index - val learnedWordEmbeddingSize: Int, // size of the learned word embedding - val charEmbeddingSize: Int, // size of the character embedding - val charRnnStateSize: Int, // size of each one of the char-level RNNs - val posTagEmbeddingSize: Int, // size of the POS tag embedding - val neTagEmbeddingSize: Int, // size of the NE tag embedding - val distanceEmbeddingSize: Int, - val distanceWindowSize: Int, // window considered for distance values (relative to predicate) - val positionEmbeddingSize: Int, - val useIsPredicate: Boolean, // if true, add a Boolean bit to indicate if current word is the predicate - val wordLookupParameters:LookupParameter, - val charLookupParameters:LookupParameter, - val charFwRnnBuilder:RnnBuilder, // RNNs for the character representation - val charBwRnnBuilder:RnnBuilder, - val posTagLookupParameters:Option[LookupParameter], - val neTagLookupParameters:Option[LookupParameter], - val distanceLookupParameters:Option[LookupParameter], - val positionLookupParameters:Option[LookupParameter], - val dropoutProb: Float) extends InitialLayer { +case class EmbeddingLayer (parameters:ParameterCollection, + w2i:Map[String, Int], // word to index + w2f:Counter[String], // word to frequency + c2i:Map[Char, Int], // character to index + tag2i:Option[Map[String, Int]], // POS tag to index + ne2i:Option[Map[String, Int]], // NE tag to index + learnedWordEmbeddingSize: Int, // size of the learned word embedding + charEmbeddingSize: Int, // size of the character embedding + charRnnStateSize: Int, // size of each one of the char-level RNNs + posTagEmbeddingSize: Int, // size of the POS tag embedding + neTagEmbeddingSize: Int, // size of the NE tag embedding + distanceEmbeddingSize: Int, + distanceWindowSize: Int, // window considered for distance values (relative to predicate) + positionEmbeddingSize: Int, + useIsPredicate: Boolean, // if true, add a Boolean bit to indicate if current word is the predicate + wordLookupParameters:LookupParameter, + charLookupParameters:LookupParameter, + charFwRnnBuilder:RnnBuilder, // RNNs for the character representation + charBwRnnBuilder:RnnBuilder, + posTagLookupParameters:Option[LookupParameter], + neTagLookupParameters:Option[LookupParameter], + distanceLookupParameters:Option[LookupParameter], + positionLookupParameters:Option[LookupParameter], + dropoutProb: Float) extends InitialLayer { lazy val constEmbedder: ConstEmbeddings = ConstEmbeddingsGlove() + override def clone(): EmbeddingLayer = { + copy( + charFwRnnBuilder = cloneBuilder(charFwRnnBuilder), + charBwRnnBuilder = cloneBuilder(charBwRnnBuilder), + ) + } + override def forward(sentence: AnnotatedSentence, doDropout: Boolean): ExpressionVector = { setCharRnnDropout(doDropout) diff --git a/main/src/main/scala/org/clulab/dynet/FinalLayer.scala b/main/src/main/scala/org/clulab/dynet/FinalLayer.scala index 3d21d6cc5..ee1da39ea 100644 --- a/main/src/main/scala/org/clulab/dynet/FinalLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/FinalLayer.scala @@ -2,7 +2,7 @@ package org.clulab.dynet import edu.cmu.dynet.{Expression, ExpressionVector} -trait FinalLayer extends Saveable { +trait FinalLayer extends Saveable with Cloneable { def forward(inputExpressions: ExpressionVector, headPositionsOpt: Option[IndexedSeq[Int]], doDropout: Boolean): ExpressionVector @@ -15,4 +15,6 @@ trait FinalLayer extends Saveable { def inference(emissionScores: Array[Array[Float]]): IndexedSeq[String] def inferenceWithScores(emissionScores: Array[Array[Float]]): IndexedSeq[IndexedSeq[(String, Float)]] + + override def clone(): FinalLayer = ??? } diff --git a/main/src/main/scala/org/clulab/dynet/ForwardLayer.scala b/main/src/main/scala/org/clulab/dynet/ForwardLayer.scala index a8ecfb176..05271198c 100644 --- a/main/src/main/scala/org/clulab/dynet/ForwardLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/ForwardLayer.scala @@ -7,16 +7,20 @@ import org.clulab.dynet.Utils.{ByLineIntBuilder, fromIndexToString, mkTransition import org.clulab.struct.Counter import org.clulab.utils.Configured -abstract class ForwardLayer (val parameters:ParameterCollection, - val inputSize: Int, - val isDual: Boolean, - val t2i: Map[String, Int], - val i2t: Array[String], - val H: Parameter, - val rootParam: Parameter, - val nonlinearity: Int, - val dropoutProb: Float) - extends FinalLayer { +abstract class ForwardLayer extends FinalLayer { + + // + // all these accessor methods will be redefined as vals in the children classes + // + def parameters:ParameterCollection + def inputSize: Int + def isDual: Boolean + def t2i: Map[String, Int] + def i2t: Array[String] + def H: Parameter + def rootParam: Parameter + def nonlinearity: Int + def dropoutProb: Float def forward(inputExpressions: ExpressionVector, headPositionsOpt: Option[IndexedSeq[Int]], @@ -91,7 +95,7 @@ abstract class ForwardLayer (val parameters:ParameterCollection, } object ForwardLayer { - val logger: Logger = LoggerFactory.getLogger(classOf[ViterbiForwardLayer]) + val logger: Logger = LoggerFactory.getLogger(classOf[ForwardLayer]) val DEFAULT_DROPOUT_PROB: Float = Utils.DEFAULT_DROPOUT_PROBABILITY diff --git a/main/src/main/scala/org/clulab/dynet/GreedyForwardLayer.scala b/main/src/main/scala/org/clulab/dynet/GreedyForwardLayer.scala index 0837a52ff..764c92046 100644 --- a/main/src/main/scala/org/clulab/dynet/GreedyForwardLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/GreedyForwardLayer.scala @@ -4,21 +4,23 @@ import java.io.PrintWriter import edu.cmu.dynet.{Dim, Expression, ExpressionVector, Parameter, ParameterCollection} import org.clulab.dynet.ForwardLayer.TYPE_GREEDY -import org.clulab.dynet.Utils.{ByLineFloatBuilder, ByLineIntBuilder, ByLineStringMapBuilder, fromIndexToString, save} +import org.clulab.dynet.Utils.{ByLineFloatBuilder, ByLineIntBuilder, ByLineStringMapBuilder, cloneBuilder, fromIndexToString, save} import ForwardLayer._ import scala.collection.mutable.ArrayBuffer -class GreedyForwardLayer (parameters:ParameterCollection, - inputSize: Int, - isDual: Boolean, - t2i: Map[String, Int], - i2t: Array[String], - H: Parameter, - rootParam: Parameter, - nonlinearity: Int, - dropoutProb: Float) - extends ForwardLayer(parameters, inputSize, isDual, t2i, i2t, H, rootParam, nonlinearity, dropoutProb) { +case class GreedyForwardLayer (override val parameters:ParameterCollection, + override val inputSize: Int, + override val isDual: Boolean, + override val t2i: Map[String, Int], + override val i2t: Array[String], + override val H: Parameter, + override val rootParam: Parameter, + override val nonlinearity: Int, + override val dropoutProb: Float) + extends ForwardLayer { + + override def clone(): GreedyForwardLayer = this override def loss(finalStates: ExpressionVector, goldLabelStrings: IndexedSeq[String]): Expression = { val goldLabels = Utils.toIds(goldLabelStrings, t2i) diff --git a/main/src/main/scala/org/clulab/dynet/InitialLayer.scala b/main/src/main/scala/org/clulab/dynet/InitialLayer.scala index 4a9c85270..933e623a8 100644 --- a/main/src/main/scala/org/clulab/dynet/InitialLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/InitialLayer.scala @@ -5,9 +5,11 @@ import edu.cmu.dynet.ExpressionVector /** * First layer that occurs in a sequence modeling architecture: goes from words to Expressions */ -trait InitialLayer extends Saveable { +trait InitialLayer extends Saveable with Cloneable { def forward(sentence: AnnotatedSentence, doDropout: Boolean): ExpressionVector def outDim:Int // output dimension + + override def clone(): InitialLayer = ??? } diff --git a/main/src/main/scala/org/clulab/dynet/IntermediateLayer.scala b/main/src/main/scala/org/clulab/dynet/IntermediateLayer.scala index 82aedd697..aa752f465 100644 --- a/main/src/main/scala/org/clulab/dynet/IntermediateLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/IntermediateLayer.scala @@ -5,10 +5,12 @@ import edu.cmu.dynet.ExpressionVector /** * Intermediate layer in a sequence modeling architecture: goes from ExpressionVector to ExpressionVector */ -trait IntermediateLayer extends Saveable { +trait IntermediateLayer extends Saveable with Cloneable { def forward(inputExpressions: ExpressionVector, doDropout: Boolean): ExpressionVector def inDim: Int def outDim: Int + + override def clone(): IntermediateLayer = ??? } diff --git a/main/src/main/scala/org/clulab/dynet/Layers.scala b/main/src/main/scala/org/clulab/dynet/Layers.scala index a32747b14..393730b2e 100644 --- a/main/src/main/scala/org/clulab/dynet/Layers.scala +++ b/main/src/main/scala/org/clulab/dynet/Layers.scala @@ -6,15 +6,29 @@ import edu.cmu.dynet.{ComputationGraph, Expression, ExpressionVector, ParameterC import org.clulab.struct.Counter import org.clulab.utils.Configured import org.clulab.dynet.Utils._ +import org.slf4j.{Logger, LoggerFactory} import scala.collection.mutable.ArrayBuffer /** * A sequence of layers that implements a complete NN architecture for sequence modeling */ -class Layers (val initialLayer: Option[InitialLayer], - val intermediateLayers: IndexedSeq[IntermediateLayer], - val finalLayer: Option[FinalLayer]) extends Saveable { +case class Layers (initialLayer: Option[InitialLayer], + intermediateLayers: IndexedSeq[IntermediateLayer], + finalLayer: Option[FinalLayer]) extends Saveable with Cloneable { + + override def clone(): Layers = { + Layers.logger.debug(s"Cloning layers: $toString...") + val clonedInitialLayer:Option[InitialLayer] = initialLayer.map(_.clone()) + val clonedIntermediateLayers:IndexedSeq[IntermediateLayer] = intermediateLayers.map(_.clone()) + val clonedFinalLayer: Option[FinalLayer] = finalLayer.map(_.clone()) + + copy( + initialLayer = clonedInitialLayer, + intermediateLayers = clonedIntermediateLayers, + finalLayer = clonedFinalLayer + ) + } def outDim: Option[Int] = { if(finalLayer.nonEmpty) { @@ -115,6 +129,8 @@ class Layers (val initialLayer: Option[InitialLayer], } object Layers { + val logger: Logger = LoggerFactory.getLogger(classOf[Layers]) + def apply(config: Configured, paramPrefix: String, parameters: ParameterCollection, @@ -203,29 +219,27 @@ object Layers { sentence: AnnotatedSentence): IndexedSeq[IndexedSeq[String]] = { val labelsPerTask = new ArrayBuffer[IndexedSeq[String]]() - DyNetSync.synchronized { // DyNet's computation graph is a static variable, so this block must be synchronized - ComputationGraph.renew() + ComputationGraph.renew() - // layers(0) contains the shared layers - if(layers(0).nonEmpty) { - val sharedStates = layers(0).forward(sentence, doDropout = false) - - for (i <- 1 until layers.length) { - val states = layers(i).forwardFrom(sharedStates, sentence.headPositions, doDropout = false) - val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) - val labels = layers(i).finalLayer.get.inference(emissionScores) - labelsPerTask += labels - } + // layers(0) contains the shared layers + if(layers(0).nonEmpty) { + val sharedStates = layers(0).forward(sentence, doDropout = false) + + for (i <- 1 until layers.length) { + val states = layers(i).forwardFrom(sharedStates, sentence.headPositions, doDropout = false) + val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) + val labels = layers(i).finalLayer.get.inference(emissionScores) + labelsPerTask += labels } + } - // no shared layer - else { - for (i <- 1 until layers.length) { - val states = layers(i).forward(sentence, doDropout = false) - val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) - val labels = layers(i).finalLayer.get.inference(emissionScores) - labelsPerTask += labels - } + // no shared layer + else { + for (i <- 1 until layers.length) { + val states = layers(i).forward(sentence, doDropout = false) + val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) + val labels = layers(i).finalLayer.get.inference(emissionScores) + labelsPerTask += labels } } @@ -237,9 +251,7 @@ object Layers { sentence: AnnotatedSentence, doDropout: Boolean): ExpressionVector = { // - // make sure this code is: - // (a) called inside a synchronized block, and - // (b) called after the computational graph is renewed (see predict below for correct usage) + // make sure this code is called after the computational graph is renewed (see predict below for correct usage) // val states = { @@ -261,14 +273,11 @@ object Layers { def predict(layers: IndexedSeq[Layers], taskId: Int, sentence: AnnotatedSentence): IndexedSeq[String] = { - val labelsForTask = - DyNetSync.synchronized { // DyNet's computation graph is a static variable, so this block must be synchronized - ComputationGraph.renew() + ComputationGraph.renew() - val states = forwardForTask(layers, taskId, sentence, doDropout = false) - val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) - layers(taskId + 1).finalLayer.get.inference(emissionScores) - } + val states = forwardForTask(layers, taskId, sentence, doDropout = false) + val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) + val labelsForTask = layers(taskId + 1).finalLayer.get.inference(emissionScores) labelsForTask } @@ -276,14 +285,11 @@ object Layers { def predictWithScores(layers: IndexedSeq[Layers], taskId: Int, sentence: AnnotatedSentence): IndexedSeq[IndexedSeq[(String, Float)]] = { - val labelsForTask = - DyNetSync.synchronized { // DyNet's computation graph is a static variable, so this block must be synchronized - ComputationGraph.renew() + ComputationGraph.renew() - val states = forwardForTask(layers, taskId, sentence, doDropout = false) - val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) - layers(taskId + 1).finalLayer.get.inferenceWithScores(emissionScores) - } + val states = forwardForTask(layers, taskId, sentence, doDropout = false) + val emissionScores: Array[Array[Float]] = Utils.emissionScoresToArrays(states) + val labelsForTask = layers(taskId + 1).finalLayer.get.inferenceWithScores(emissionScores) labelsForTask } diff --git a/main/src/main/scala/org/clulab/dynet/LayersSupplier.scala b/main/src/main/scala/org/clulab/dynet/LayersSupplier.scala new file mode 100644 index 000000000..529fc4d23 --- /dev/null +++ b/main/src/main/scala/org/clulab/dynet/LayersSupplier.scala @@ -0,0 +1,9 @@ +package org.clulab.dynet + +import java.util.function.Supplier + +class LayersSupplier(val referenceLayers: IndexedSeq[Layers]) extends Supplier[IndexedSeq[Layers]] { + override def get(): IndexedSeq[Layers] = { + referenceLayers.map(_.clone()) + } +} diff --git a/main/src/main/scala/org/clulab/dynet/Metal.scala b/main/src/main/scala/org/clulab/dynet/Metal.scala index e058435e3..e76b3b8e8 100644 --- a/main/src/main/scala/org/clulab/dynet/Metal.scala +++ b/main/src/main/scala/org/clulab/dynet/Metal.scala @@ -24,10 +24,17 @@ import Metal._ */ class Metal(val taskManagerOpt: Option[TaskManager], val parameters: ParameterCollection, - modelOpt: Option[IndexedSeq[Layers]]) { + modelOpt: Option[IndexedSeq[Layers]], + val multiThreaded: Boolean) { // One Layers object per task; model(0) contains the Layers shared between all tasks (if any) protected lazy val model: IndexedSeq[Layers] = modelOpt.getOrElse(initialize()) + // Model to be used only during inference, and only if the configuration indicates multi-threaded execution + protected lazy val multiThreadedModel: ThreadLocal[IndexedSeq[Layers]] = + ThreadLocal.withInitial(new LayersSupplier(model)) + + protected def getInferenceModel: IndexedSeq[Layers] = if(multiThreaded) multiThreadedModel.get() else model + // Use this carefully. That is, only when taskManagerOpt.isDefined def taskManager: TaskManager = { assert(taskManagerOpt.isDefined) @@ -287,7 +294,7 @@ class Metal(val taskManagerOpt: Option[TaskManager], val sentence = as._1 val goldLabels = as._2 - val preds = Layers.predict(model, taskId, sentence) + val preds = Layers.predict(getInferenceModel, taskId, sentence) val sc = SeqScorer.f1(goldLabels, preds) scoreCountsByLabel.incAll(sc) @@ -314,15 +321,15 @@ class Metal(val taskManagerOpt: Option[TaskManager], // this only supports basic tasks for now def predictJointly(sentence: AnnotatedSentence): IndexedSeq[IndexedSeq[String]] = { - Layers.predictJointly(model, sentence) + Layers.predictJointly(getInferenceModel, sentence) } def predict(taskId: Int, sentence: AnnotatedSentence): IndexedSeq[String] = { - Layers.predict(model, taskId, sentence) + Layers.predict(getInferenceModel, taskId, sentence) } def predictWithScores(taskId: Int, sentence: AnnotatedSentence): IndexedSeq[IndexedSeq[(String, Float)]] = { - Layers.predictWithScores(model, taskId, sentence) + Layers.predictWithScores(getInferenceModel, taskId, sentence) } def test(): Unit = { @@ -396,22 +403,22 @@ object Metal { def apply(modelFilenamePrefix: String, taskManager: TaskManager): Metal = { val parameters = new ParameterCollection() val model = Metal.load(parameters, modelFilenamePrefix) - val mtl = new Metal(Some(taskManager), parameters, Some(model)) + val mtl = new Metal(Some(taskManager), parameters, Some(model), multiThreaded = false) mtl } def apply(modelFilenamePrefix: String): Metal = { val parameters = new ParameterCollection() val model = Metal.load(parameters, modelFilenamePrefix) - val mtl = new Metal(None, parameters, Some(model)) + val mtl = new Metal(None, parameters, Some(model), multiThreaded = true) mtl } def main(args: Array[String]): Unit = { val props = StringUtils.argsToProperties(args) - initializeDyNet() // autoBatch = true, mem = "2048,2048,2048,2048") // mem = "1660,1664,2496,1400") if(props.containsKey("train")) { + initializeDyNet(train = true) // autoBatch = true, mem = "2048,2048,2048,2048") // mem = "1660,1664,2496,1400") assert(props.containsKey("conf")) val configName = props.getProperty("conf") val config = ConfigFactory.load(configName) @@ -419,11 +426,12 @@ object Metal { val taskManager = new TaskManager(config) val modelName = props.getProperty("train") - val mtl = new Metal(Some(taskManager), parameters, None) + val mtl = new Metal(Some(taskManager), parameters, None, multiThreaded = false) mtl.train(modelName) } else if(props.containsKey("test")) { + initializeDyNet(train = false) assert(props.containsKey("conf")) val configName = props.getProperty("conf") val config = ConfigFactory.load(configName) @@ -435,6 +443,7 @@ object Metal { } else if(props.containsKey("shell")) { + initializeDyNet(train = false) val modelName = props.getProperty("shell") val mtl = Metal(modelName) val shell = new MetalShell(mtl) diff --git a/main/src/main/scala/org/clulab/dynet/RnnLayer.scala b/main/src/main/scala/org/clulab/dynet/RnnLayer.scala index b70747259..c63c15148 100644 --- a/main/src/main/scala/org/clulab/dynet/RnnLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/RnnLayer.scala @@ -8,19 +8,28 @@ import scala.collection.mutable.ArrayBuffer import org.clulab.dynet.Utils._ import org.clulab.utils.Configured +import scala.collection.mutable + /** * This layer applies a biLSTM over the sequence of Expressions produced by a previous layer * @author Mihai */ -class RnnLayer (val parameters:ParameterCollection, - val inputSize: Int, - val numLayers: Int, - val rnnStateSize: Int, - val useHighwayConnections: Boolean, - val rnnType: String, // "gru" or "lstm" - val wordFwRnnBuilder:RnnBuilder, - val wordBwRnnBuilder:RnnBuilder, - val dropoutProb: Float) extends IntermediateLayer { +case class RnnLayer (parameters:ParameterCollection, + inputSize: Int, + numLayers: Int, + rnnStateSize: Int, + useHighwayConnections: Boolean, + rnnType: String, // "gru" or "lstm" + wordFwRnnBuilder:RnnBuilder, + wordBwRnnBuilder:RnnBuilder, + dropoutProb: Float) extends IntermediateLayer { + + override def clone(): RnnLayer = { + copy( + wordFwRnnBuilder = cloneBuilder(wordFwRnnBuilder), + wordBwRnnBuilder = cloneBuilder(wordBwRnnBuilder) + ) + } override def forward(inputExpressions: ExpressionVector, doDropout: Boolean): ExpressionVector = { setRnnDropout(wordFwRnnBuilder, dropoutProb, doDropout) diff --git a/main/src/main/scala/org/clulab/dynet/Utils.scala b/main/src/main/scala/org/clulab/dynet/Utils.scala index 85c633478..5ee9ba97f 100644 --- a/main/src/main/scala/org/clulab/dynet/Utils.scala +++ b/main/src/main/scala/org/clulab/dynet/Utils.scala @@ -6,6 +6,7 @@ import edu.cmu.dynet.Expression.{concatenate, input, logSumExp, lookup, pick, pi import edu.cmu.dynet._ import org.clulab.embeddings.WordEmbeddingMap import org.clulab.fatdynet.utils.BaseTextLoader +import org.clulab.fatdynet.utils.Initializer import org.clulab.struct.{Counter, MutableNumber} import org.clulab.utils.Serializer import org.slf4j.{Logger, LoggerFactory} @@ -31,28 +32,26 @@ object Utils { val LOG_MIN_VALUE: Float = -10000 - val DEFAULT_DROPOUT_PROBABILITY = 0f // no dropout by default + val DEFAULT_DROPOUT_PROBABILITY = 0f // no dropout by default' - private var IS_DYNET_INITIALIZED = false + def initializeDyNetForInference(): Unit = { + initializeDyNet(train = false) + } - def initializeDyNet(autoBatch: Boolean = false, mem: String = ""): Unit = { - DyNetSync.synchronized { - if (!IS_DYNET_INITIALIZED) { - logger.debug("Initializing DyNet...") + def initializeDyNet(train: Boolean, autoBatch: Boolean = false, mem: String = ""): Unit = { + logger.debug("Initializing DyNet...") - val params = new mutable.HashMap[String, Any]() - params += "random-seed" -> RANDOM_SEED - params += "weight-decay" -> WEIGHT_DECAY - if (autoBatch) { - params += "autobatch" -> 1 - params += "dynet-mem" -> mem - } + val map = Map( + Initializer.RANDOM_SEED -> RANDOM_SEED, + Initializer.WEIGHT_DECAY -> WEIGHT_DECAY, + Initializer.FORWARD_ONLY -> { if (train) 0 else 1 }, + Initializer.AUTOBATCH -> { if(autoBatch) 1 else 0 }, + Initializer.DYNAMIC_MEM -> ! train, + Initializer.DYNET_MEM -> { if(mem != null && mem.length > 0) mem else "2048" } + ) - Initialize.initialize(params.toMap) - logger.debug("DyNet initialization complete.") - IS_DYNET_INITIALIZED = true - } - } + Initializer.initialize(map) + logger.debug("DyNet initialization complete.") } def fromIndexToString(s2i: Map[String, Int]): Array[String] = { @@ -846,6 +845,13 @@ object Utils { expression } } + + def cloneBuilder(builder: RnnBuilder): RnnBuilder = { + val newBuilder = builder.clone() + + newBuilder.newGraph() + newBuilder + } } class Utils diff --git a/main/src/main/scala/org/clulab/dynet/ViterbiForwardLayer.scala b/main/src/main/scala/org/clulab/dynet/ViterbiForwardLayer.scala index 4ec1d274d..0ac204a81 100644 --- a/main/src/main/scala/org/clulab/dynet/ViterbiForwardLayer.scala +++ b/main/src/main/scala/org/clulab/dynet/ViterbiForwardLayer.scala @@ -6,17 +6,19 @@ import edu.cmu.dynet.{Dim, Expression, ExpressionVector, FloatVector, LookupPara import org.clulab.dynet.Utils.{ByLineFloatBuilder, ByLineIntBuilder, ByLineStringMapBuilder, LOG_MIN_VALUE, START_TAG, STOP_TAG, fromIndexToString, mkTransitionMatrix, save} import ForwardLayer._ -class ViterbiForwardLayer(parameters:ParameterCollection, - inputSize: Int, - isDual: Boolean, - t2i: Map[String, Int], - i2t: Array[String], - H: Parameter, - val T: LookupParameter, // transition matrix for Viterbi; T[i][j] = transition *to* i *from* j, one per task - rootParam: Parameter, - nonlinearity: Int, - dropoutProb: Float) - extends ForwardLayer(parameters, inputSize, isDual, t2i, i2t, H, rootParam, nonlinearity, dropoutProb) { +case class ViterbiForwardLayer(override val parameters:ParameterCollection, + override val inputSize: Int, + override val isDual: Boolean, + override val t2i: Map[String, Int], + override val i2t: Array[String], + override val H: Parameter, + T: LookupParameter, // transition matrix for Viterbi; T[i][j] = transition *to* i *from* j, one per task + override val rootParam: Parameter, + override val nonlinearity: Int, + override val dropoutProb: Float) + extends ForwardLayer { + + override def clone(): ViterbiForwardLayer = this // call this *before* training a model, but not on a saved model def initializeTransitions(): Unit = { diff --git a/main/src/main/scala/org/clulab/processors/clu/CluShell.scala b/main/src/main/scala/org/clulab/processors/clu/CluShell.scala index 6b7f7e868..67853f2e2 100644 --- a/main/src/main/scala/org/clulab/processors/clu/CluShell.scala +++ b/main/src/main/scala/org/clulab/processors/clu/CluShell.scala @@ -25,7 +25,7 @@ class CluShell extends Shell { object CluShell { def main(args:Array[String]): Unit = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) val sh = new CluShell sh.shell() } diff --git a/main/src/main/scala/org/clulab/processors/clu/ProcessFile.scala b/main/src/main/scala/org/clulab/processors/clu/ProcessFile.scala new file mode 100644 index 000000000..fe1bfacf3 --- /dev/null +++ b/main/src/main/scala/org/clulab/processors/clu/ProcessFile.scala @@ -0,0 +1,14 @@ +package org.clulab.processors.clu + +import org.clulab.dynet.Utils + +object ProcessFile extends App { + val fileName = args(0) + lazy val proc = new CluProcessor() + val text = io.Source.fromFile(fileName).mkString + //println(text) + + Utils.initializeDyNetForInference() + val doc = proc.mkDocument(text) + proc.annotate(doc) +} diff --git a/main/src/main/scala/org/clulab/sequences/LexiconNERShell.scala b/main/src/main/scala/org/clulab/sequences/LexiconNERShell.scala index e9499080c..8c1422726 100644 --- a/main/src/main/scala/org/clulab/sequences/LexiconNERShell.scala +++ b/main/src/main/scala/org/clulab/sequences/LexiconNERShell.scala @@ -21,7 +21,7 @@ class LexiconNERShell(val lexiconNer: LexiconNER) extends Shell { } object LexiconNERShell extends App { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) val kb = args(0) // pass the KB file name as the first argument val lexiconNer = LexiconNER(Seq(kb)) val shell = new LexiconNERShell(lexiconNer) diff --git a/main/src/main/scala/org/clulab/utils/ToEnhancedSemanticRoles.scala b/main/src/main/scala/org/clulab/utils/ToEnhancedSemanticRoles.scala index e83831d5f..0c3e47643 100644 --- a/main/src/main/scala/org/clulab/utils/ToEnhancedSemanticRoles.scala +++ b/main/src/main/scala/org/clulab/utils/ToEnhancedSemanticRoles.scala @@ -77,8 +77,8 @@ object ToEnhancedSemanticRoles { val modifier = leftRole.destination if(modifier != right) { // no self loops val label = leftRole.relation - val e = Edge(right, modifier, label) - if (!rightRoles.contains(e)) { + if(! contains(rightRoles, right, label)) { + val e = Edge(right, modifier, label) toAdd += e } } @@ -89,8 +89,8 @@ object ToEnhancedSemanticRoles { val modifier = rightRole.destination if(modifier != left) { // no self loops val label = rightRole.relation - val e = Edge(left, modifier, label) - if (!leftRoles.contains(e)) { + if(! contains(leftRoles, left, label)) { + val e = Edge(left, modifier, label) toAdd += e } } @@ -100,8 +100,17 @@ object ToEnhancedSemanticRoles { for(e <- toAdd) rolesIndex.addEdge(e.source, e.destination, e.relation) } + private def contains(roles: Set[Edge[String]], head: Int, label: String): Boolean = { + for(role <- roles) { + if(role.source == head && role.relation == label) { + return true + } + } + false + } + /** - * Propagates conjoined subjects and objects to same verb (works for SD and UD) + * Propagates conjoined subjects and objects to same verb * Paul and Mary are reading a book => A0 from 4 to 0 and from 4 to 2 * John is reading a book and a newspaper => A1 from 2 to 4 and from 2 to 7 */ diff --git a/main/src/test/scala/org/clulab/dynet/TestConstEmbeddingsGlove.scala b/main/src/test/scala/org/clulab/dynet/TestConstEmbeddingsGlove.scala index a00889d9d..4a07139fd 100644 --- a/main/src/test/scala/org/clulab/dynet/TestConstEmbeddingsGlove.scala +++ b/main/src/test/scala/org/clulab/dynet/TestConstEmbeddingsGlove.scala @@ -5,7 +5,7 @@ import org.scalatest.{FlatSpec, Matchers} class TestConstEmbeddingsGlove extends FlatSpec with Matchers { lazy val embeddings = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) ConstEmbeddingsGlove("/test_vectors.txt", true) } diff --git a/main/src/test/scala/org/clulab/processors/TestCluProcessor.scala b/main/src/test/scala/org/clulab/processors/TestCluProcessor.scala index 238be1348..ec3127452 100644 --- a/main/src/test/scala/org/clulab/processors/TestCluProcessor.scala +++ b/main/src/test/scala/org/clulab/processors/TestCluProcessor.scala @@ -11,7 +11,7 @@ import org.scalatest.{FlatSpec, Matchers} */ class TestCluProcessor extends FlatSpec with Matchers { val proc = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) new CluProcessor() } diff --git a/main/src/test/scala/org/clulab/processors/TestDepGraphSizes.scala b/main/src/test/scala/org/clulab/processors/TestDepGraphSizes.scala index 6796cdd81..0510e1da9 100644 --- a/main/src/test/scala/org/clulab/processors/TestDepGraphSizes.scala +++ b/main/src/test/scala/org/clulab/processors/TestDepGraphSizes.scala @@ -9,7 +9,7 @@ import org.scalatest.{FlatSpec, Matchers} /** Makes sure that CluProcessor produces dependency graphs of correct sizes */ class TestDepGraphSizes extends FlatSpec with Matchers { lazy val proc = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) new CluProcessor() } diff --git a/main/src/test/scala/org/clulab/processors/TestEnhancedSemanticRoles.scala b/main/src/test/scala/org/clulab/processors/TestEnhancedSemanticRoles.scala index edf7801c7..c3d4145c1 100644 --- a/main/src/test/scala/org/clulab/processors/TestEnhancedSemanticRoles.scala +++ b/main/src/test/scala/org/clulab/processors/TestEnhancedSemanticRoles.scala @@ -6,7 +6,7 @@ import org.scalatest.{FlatSpec, Matchers} class TestEnhancedSemanticRoles extends FlatSpec with Matchers { val proc = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) new CluProcessor() } @@ -35,7 +35,7 @@ class TestEnhancedSemanticRoles extends FlatSpec with Matchers { doc.sentences.head.enhancedSemanticRoles.get.hasEdge(3, 6, "A1") should be(true) } - it should "propagate subjects and objects in conjoined predicates" in { + it should "propagate subjects and objects in conjoined predicates (1)" in { val doc = proc.annotate("The store buys and sells cameras.") doc.sentences.head.enhancedSemanticRoles.get.hasEdge(2, 1, "A0") should be(true) @@ -44,6 +44,31 @@ class TestEnhancedSemanticRoles extends FlatSpec with Matchers { doc.sentences.head.enhancedSemanticRoles.get.hasEdge(4, 5, "A1") should be(true) } + it should "propagate subjects and objects in conjoined predicates (2)" in { + val doc = proc.annotate("John and Mary eat cake and drink wine.") + + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(3, 0, "A0") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(3, 2, "A0") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(6, 0, "A0") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(6, 2, "A0") should be(true) + + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(3, 4, "A1") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(3, 7, "A1") should be(false) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(6, 4, "A1") should be(false) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(6, 7, "A1") should be(true) + } + + it should "propagate subjects and objects in conjoined predicates (3)" in { + val doc = proc.annotate("The store buys pizzas and sells cameras.") + + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(2, 1, "A0") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(2, 3, "A1") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(5, 1, "A0") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(5, 6, "A1") should be(true) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(2, 6, "A1") should be(false) + doc.sentences.head.enhancedSemanticRoles.get.hasEdge(5, 3, "A1") should be(false) + } + it should "apply the deterministic predicate corrections" in { val doc = proc.annotate("The price of water trucking.") diff --git a/main/src/test/scala/org/clulab/processors/TestUniversalEnhancedDependencies.scala b/main/src/test/scala/org/clulab/processors/TestUniversalEnhancedDependencies.scala index aca3ec93a..36cebb66f 100644 --- a/main/src/test/scala/org/clulab/processors/TestUniversalEnhancedDependencies.scala +++ b/main/src/test/scala/org/clulab/processors/TestUniversalEnhancedDependencies.scala @@ -6,7 +6,7 @@ import org.scalatest.{FlatSpec, Matchers} class TestUniversalEnhancedDependencies extends FlatSpec with Matchers { val proc = { - Utils.initializeDyNet() + Utils.initializeDyNet(train = false) new CluProcessor() }