Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 30 additions & 9 deletions src/main/scala/com/herminiogarcia/shexml/MappingLauncher.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,13 +6,15 @@ import com.herminiogarcia.shexml.helper.{OrphanBNodeRemover, ParallelExecutionCo
import com.herminiogarcia.shexml.parser.ASTCreatorVisitor
import com.herminiogarcia.shexml.shex._
import com.herminiogarcia.shexml.visitor.{PushedOrPoppedValueSearchVisitor, RDFGeneratorVisitor, RMLGeneratorVisitor, VarTableBuilderVisitor}
import com.typesafe.scalalogging.Logger
import org.antlr.v4.runtime.{CharStreams, CommonTokenStream}
import org.apache.jena.query.{Dataset, DatasetFactory}
import org.apache.jena.riot.{RDFDataMgr, RDFFormat, RDFLanguages}
import com.typesafe.scalalogging.Logger

import java.io.ByteArrayOutputStream
import scala.collection.JavaConverters._
import java.nio.file.Path
import java.util.concurrent.ConcurrentLinkedQueue
import scala.collection.JavaConverters._
import scala.collection.mutable

/**
Expand All @@ -21,7 +23,21 @@ import scala.collection.mutable
class MappingLauncher(val username: String = "", val password: String = "", drivers: String = "",
val inferenceDatatype: Boolean = false,
val normaliseURIs: Boolean = false,
val parallelCollectionConfigurator: ParallelExecutionConfigurator = new ParallelExecutionConfigurator(Map(), None)) {
val parallelCollectionConfigurator: ParallelExecutionConfigurator = new ParallelExecutionConfigurator(Map(), None),
val basePath: Path = Path.of("")) {

/**
* Java compatibility constructor.
*/
def this(
username: String,
password: String,
drivers: String,
inferenceDatatype: Boolean,
normaliseURIs: Boolean,
parallelCollectionConfigurator: ParallelExecutionConfigurator) = {
this(username, password, drivers, inferenceDatatype, normaliseURIs, parallelCollectionConfigurator, Path.of(""))
}

private val logger = Logger[MappingLauncher]

Expand Down Expand Up @@ -158,7 +174,8 @@ class MappingLauncher(val username: String = "", val password: String = "", driv
pushedOrPoppedFieldsPresent = pushedOrPoppedFields,
inferenceDatatype = inferenceDatatype,
normaliseURIs = normaliseURIs,
parallelCollectionConfigurator = parallelCollectionConfigurator).doVisit(ast, null)
parallelCollectionConfigurator = parallelCollectionConfigurator,
basePath = basePath).doVisit(ast, null)
//val in = new ByteArrayInputStream(output.toString().getBytes)
//val model = ModelFactory.createDefaultModel()
//model.read(in, null, "TURTLE")
Expand All @@ -167,7 +184,7 @@ class MappingLauncher(val username: String = "", val password: String = "", driv

private def generateResultingRML(ast: AST, varTable: mutable.HashMap[Variable, VarResult], prettify: Boolean): Dataset = {
val output = DatasetFactory.create()
new RMLGeneratorVisitor(output, varTable.toMap, prettify, username, password).doVisit(ast, null)
new RMLGeneratorVisitor(output, varTable.toMap, prettify, username, password, basePath = basePath).doVisit(ast, null)
output
}

Expand All @@ -179,7 +196,9 @@ class MappingLauncher(val username: String = "", val password: String = "", driv
pushedOrPoppedFieldsPresent = searchForPushedOrPoppedFields(ast),
inferenceDatatype = inferenceDatatype,
normaliseURIs = normaliseURIs,
parallelCollectionConfigurator = parallelCollectionConfigurator).doVisit(ast, null)
parallelCollectionConfigurator = parallelCollectionConfigurator,
basePath = basePath
).doVisit(ast, null)
}

private def generateShapeMaps(ast: AST, varTable: mutable.HashMap[Variable, VarResult]): List[ShapeMapInference] = {
Expand All @@ -191,7 +210,9 @@ class MappingLauncher(val username: String = "", val password: String = "", driv
pushedOrPoppedFieldsPresent = searchForPushedOrPoppedFields(ast),
inferenceDatatype = inferenceDatatype,
normaliseURIs = normaliseURIs,
parallelCollectionConfigurator = parallelCollectionConfigurator).doVisit(ast, null)
parallelCollectionConfigurator = parallelCollectionConfigurator,
basePath = basePath
).doVisit(ast, null)
shapeMapTable.asScala.toList
}

Expand All @@ -206,13 +227,13 @@ class MappingLauncher(val username: String = "", val password: String = "", driv
private def searchForPushedOrPoppedFields(ast: AST): Boolean = new PushedOrPoppedValueSearchVisitor().doVisit(ast, null)

private def resolveImports(mappingRules: String): String = {
val sourceHelper = new SourceHelper()
val sourceHelper = SourceHelper()
val regex = "[Ii][Mm][Pp][Oo][Rr][Tt]\\s*<(.+)>".r
regex.replaceAllIn(mappingRules, matchedPart => {
val importSource = matchedPart.group(1)
val loadedSource =
if(importSource.contains("://")) sourceHelper.getURLContent(importSource)
else sourceHelper.getContentFromRelativePath(importSource)
else sourceHelper.getContentFromRelativePath(importSource, basePath)
java.util.regex.Matcher.quoteReplacement(loadedSource.fileContent)
})
}
Expand Down
28 changes: 16 additions & 12 deletions src/main/scala/com/herminiogarcia/shexml/helper/SourceHelper.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,40 +2,44 @@ package com.herminiogarcia.shexml.helper

import com.herminiogarcia.shexml.helper.SourceHelper.{saveFileResult, searchFileResult}

import java.nio.charset.StandardCharsets
import java.nio.file.Path
import scala.collection.mutable

/**
* Created by herminio on 21/2/18.
*/
class SourceHelper {
case class SourceHelper() {

def getURLContent(url: String): LoadedSource = searchFileResult(url) match {
case Some(result) => result
case None =>
val parsedURL = new java.net.URL(url)
val file = scala.io.Source.fromURL(parsedURL, "UTF-8")
val file = scala.io.Source.fromURL(parsedURL, StandardCharsets.UTF_8.toString)
try {
val content = LoadedSource(file.mkString, url)
saveFileResult(url, content)
content
} finally { file.close() }
}

def getContentFromRelativePath(path: String): LoadedSource = searchFileResult(path) match {
case Some(result) => result
case None =>
val file = scala.io.Source.fromFile(path, "UTF-8")
try {
val content = LoadedSource(file.mkString, path)
saveFileResult(path, content)
content
} finally { file.close() }
def getContentFromRelativePath(path: String, base: Path = Path.of("")): LoadedSource = {
val fullPath = base.resolve(path).normalize().toString
searchFileResult(fullPath) match {
case Some(result) => result
case None =>
val file = scala.io.Source.fromFile(fullPath, StandardCharsets.UTF_8.toString)
try {
val content = LoadedSource(file.mkString, fullPath)
saveFileResult(fullPath, content)
content
} finally { file.close() }
}
}

def getStdinContents(): LoadedSource = {
LoadedSource(scala.io.Source.stdin.mkString, "-")
}

}

object SourceHelper {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class QuerySearcher(val varTable: Map[Variable, VarResult]) {
private def getURLContents(u: URL): QueryClause = {
val parts = u.value.split('.')
val extension = parts(parts.length - 1)
val file = new SourceHelper().getURLContent(u.value)
val file = SourceHelper().getURLContent(u.value)
if(extension == "xpath") {
XmlPath(file.fileContent)
} else if(extension == "jsonpath") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import org.apache.jena.util.SplitIRI

import scala.collection.concurrent
import java.io.{File, StringReader}
import java.nio.file.Path
import java.sql.DriverManager
import java.util.concurrent.ConcurrentLinkedQueue
import javax.xml.transform.stream.StreamSource
Expand All @@ -35,7 +36,8 @@ class RDFGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult],
pushedOrPoppedFieldsPresent: Boolean = true,
inferenceDatatype: Boolean = false,
normaliseURIs: Boolean = false,
parallelCollectionConfigurator: ParallelExecutionConfigurator = new ParallelExecutionConfigurator(Map(), None))
parallelCollectionConfigurator: ParallelExecutionConfigurator = new ParallelExecutionConfigurator(Map(), None),
basePath: Path)
extends DefaultVisitor[Any, Any] with JdbcDriverRegistry {

protected val prefixTable = concurrent.TrieMap[String, String](("rdf:", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"))
Expand All @@ -49,6 +51,7 @@ class RDFGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult],
protected val xpathQueryResultsCache = new XpathQueryResultsCache(pushedOrPoppedFieldsPresent)
protected val xmlDocumentCache = new XMLDocumentCache()
protected val functionHubExecuterCache = new FunctionHubExecutorCache()
protected val sourceHelper: SourceHelper = SourceHelper()
protected val defaultModel = dataset.getDefaultModel
protected val jsonPathConfiguration = Configuration.defaultConfiguration()
.addOptions(com.jayway.jsonpath.Option.ALWAYS_RETURN_LIST)
Expand Down Expand Up @@ -614,7 +617,7 @@ class RDFGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult],
else if(url.contains('*'))
throw new Exception("* wildcard not allowed over remote files")
else
List(new SourceHelper().getURLContent(url))
List(sourceHelper.getURLContent(url))

case RelativePath(path) =>
if(isRDFSource(path)) {
Expand All @@ -625,11 +628,11 @@ class RDFGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult],
List(LoadedSource("", fileProtocol + fileAbsolutePath))
}
else if(path.contains('*')) getAllFilesContents(path)
else List(new SourceHelper().getContentFromRelativePath(path))
else List(sourceHelper.getContentFromRelativePath(path, basePath))

case JdbcURL(url) => List(LoadedSource("", url))

case Stdin() => List(new SourceHelper().getStdinContents())
case Stdin() => List(sourceHelper.getStdinContents())


case default => visit(default, optionalArgument)
Expand Down Expand Up @@ -1007,10 +1010,11 @@ class RDFGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult],
val fileBeginning = slices(0).splitAt(slices(0).lastIndexOf("/"))._2.replace("/", "")
val fileEnding = slices(1).splitAt(slices(1).lastIndexOf("."))._1
val fileExtension = slices(1).splitAt(slices(1).lastIndexOf("."))._2
val files = new File(path).listFiles().filter(_.isFile)
val normPath = basePath.resolve(path).normalize()
val files = normPath.toFile.listFiles().filter(_.isFile)
.filter(_.getName.endsWith(fileEnding + fileExtension)).filter(_.getName.startsWith(fileBeginning))
val fileProtocol = if(path.startsWith("/")) "file://" else "file:///"
files.map(file => new SourceHelper().getURLContent(fileProtocol + file.getAbsolutePath.replaceAll("\\\\", "/"))).toList
files.map(file => sourceHelper.getURLContent(fileProtocol + file.getAbsolutePath.replaceAll("\\\\", "/"))).toList
}

private def visitAction(actionOrLiteral: ActionOrLiteral, predicateObjectsList: List[Any], optionalArgument: Any): List[Result] = actionOrLiteral match {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@ import com.typesafe.scalalogging.Logger
import org.apache.jena.query.Dataset
import org.apache.jena.rdf.model.{Resource, Statement}

class RMLGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult], prettify: Boolean ,username: String, password: String)
extends RDFGeneratorVisitor(dataset, varTable, username, password) with JdbcDriverRegistry {
import java.nio.file.Path

class RMLGeneratorVisitor(dataset: Dataset, varTable: Map[Variable, VarResult], prettify: Boolean ,username: String, password: String, basePath: Path)
extends RDFGeneratorVisitor(dataset, varTable, username, password, basePath = basePath) with JdbcDriverRegistry {

private val mapPrefix = "http://mapping.example.com/"
private val rmlPrefix = "http://semweb.mmlab.be/ns/rml#"
Expand Down
2 changes: 1 addition & 1 deletion src/test/scala/com/herminiogarcia/shexml/FilmsStdin.scala
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class FilmsStdin extends AnyFunSuite

override def beforeAll(configMap: ConfigMap): Unit = {
super.beforeAll(configMap)
val stream = new ByteArrayInputStream(new SourceHelper().getContentFromRelativePath("./src/test/resources/filmsAlt.json").fileContent.getBytes())
val stream = new ByteArrayInputStream(SourceHelper().getContentFromRelativePath("./src/test/resources/filmsAlt.json").fileContent.getBytes())
System.setIn(stream)
stream.close()
output = mappingLauncher.launchMapping(example).getDefaultModel
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ class MultipleElementIteratorExpressionWithCSVFromStdinTest extends AnyFunSuite

override def beforeAll(configMap: ConfigMap): Unit = {
super.beforeAll(configMap)
val stream = new ByteArrayInputStream(new SourceHelper().getURLContent("https://rawgit.com/herminiogg/ShExML/enhancement-%239/src/test/resources/films.csv").fileContent.getBytes())
val stream = new ByteArrayInputStream(SourceHelper().getURLContent("https://rawgit.com/herminiogg/ShExML/enhancement-%239/src/test/resources/films.csv").fileContent.getBytes())
System.setIn(stream)
stream.close()
output = mappingLauncher.launchMapping(example).getDefaultModel
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package com.herminiogarcia.shexml
import com.herminiogarcia.shexml.helper.ParallelExecutionConfigurator
import org.scalatest.{BeforeAndAfterAllConfigMap, ConfigMap, TestSuite}

import java.nio.file.Path


trait ParallelConfigInferenceDatatypesNormaliseURIsFixture extends BeforeAndAfterAllConfigMap with MappingLauncherInitializer { this: TestSuite =>
var mappingLauncher: MappingLauncher = _
Expand Down Expand Up @@ -49,6 +51,17 @@ trait ParallelConfigSparql extends BeforeAndAfterAllConfigMap with MappingLaunch
}
}

trait ParallelConfigBasePath extends BeforeAndAfterAllConfigMap with MappingLauncherInitializer { this: TestSuite =>
var mappingLauncher: MappingLauncher = _

def relativeBasePath: Path

override def beforeAll(configMap: ConfigMap): Unit = {
mappingLauncher = configMapToParallelConfiguration(configMap).parallelConfigurationBasePath(relativeBasePath)
super.beforeAll(configMap)
}
}

trait ParallelConfigValidation extends BeforeAndAfterAllConfigMap with MappingLauncherInitializer { this: TestSuite =>
var parallelConfiguration: ParallelExecutionConfigurator = _

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package com.herminiogarcia.shexml

import com.herminiogarcia.shexml.helper.ParallelExecutionConfigurator

import java.nio.file.Path

class ParallelConfigurations(val parallelConfiguration: ParallelExecutionConfigurator) {

val parallelConfigurationInferenceDatatypesNormaliseURIs = new MappingLauncher(
Expand Down Expand Up @@ -34,6 +36,11 @@ class ParallelConfigurations(val parallelConfiguration: ParallelExecutionConfigu
"root",
parallelCollectionConfigurator = parallelConfiguration
)

def parallelConfigurationBasePath(basePath: Path) = new MappingLauncher(
parallelCollectionConfigurator = parallelConfiguration,
basePath = basePath
)
}

object ParallelConfigurations {
Expand Down
Loading