From afe85db2f652546807370c7bb3636115ed240215 Mon Sep 17 00:00:00 2001 From: PogI Date: Wed, 13 Apr 2016 09:45:58 +0400 Subject: [PATCH 1/3] [BACKLOG-6461] Oracle repository: Impossible to create CSV datasource where column names in csv contain - and / which get converted to text, HYPHEN and DIVIDED_BY, causing column name in table to be too long [ORA-00972 - Identifier too long]. --- .../service/agile/CsvTransformGenerator.java | 167 ++++++++++- .../agile/StagingTransformGenerator.java | 4 + .../agile/CsvTransformGeneratorIT.java | 270 +++++++++++++++++- .../agile/CsvTransformGeneratorTest.java | 78 +++++ 4 files changed, 508 insertions(+), 11 deletions(-) create mode 100644 test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java diff --git a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java index d000fc8d0..b037748cd 100644 --- a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java +++ b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGenerator.java @@ -12,19 +12,28 @@ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. * See the GNU Lesser General Public License for more details. * -* Copyright (c) 2002-2013 Pentaho Corporation.. All rights reserved. +* Copyright (c) 2002-2016 Pentaho Corporation.. All rights reserved. */ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; import java.io.File; +import java.sql.DatabaseMetaData; +import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.pentaho.di.core.database.Database; import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.exception.KettleTransException; +import org.pentaho.di.core.row.RowMetaInterface; import org.pentaho.di.core.row.ValueMeta; +import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.StepErrorMeta; import org.pentaho.di.trans.step.StepMeta; @@ -33,8 +42,8 @@ import org.pentaho.di.trans.steps.textfileinput.TextFileInputField; import org.pentaho.platform.dataaccess.datasource.wizard.models.ColumnInfo; import org.pentaho.platform.dataaccess.datasource.wizard.models.CsvFileInfo; -import org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats; import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; +import org.pentaho.platform.dataaccess.datasource.wizard.sources.csv.FileTransformStats; import org.pentaho.platform.engine.core.system.PentahoSystem; public class CsvTransformGenerator extends StagingTransformGenerator { @@ -45,6 +54,8 @@ public class CsvTransformGenerator extends StagingTransformGenerator { private static final String SELECT_VALUES = "select"; //$NON-NLS-1$ + private static final String CUT_LONG_NAMES = "cutLongNames"; //$NON-NLS-1$ + public static final String DEFAULT_RELATIVE_UPLOAD_FILE_PATH = File.separatorChar + "system" + File.separatorChar + "metadata" + File.separatorChar + "csvfiles" + File.separatorChar; //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$ @@ -106,6 +117,14 @@ protected StepMeta[] getSteps( TransMeta transMeta ) { createHop(steps.get(steps.size()-2), step, transMeta); } */ + + final int targetDatabaseMaxColumnNameLength = getMaxColumnNameLength(); + StepMeta cutLongNamesStep = createCutLongNamesStep( transMeta, steps, targetDatabaseMaxColumnNameLength, CUT_LONG_NAMES ); + if ( cutLongNamesStep != null ) { + steps.add( cutLongNamesStep ); + createHop( steps.get( steps.size() - 2 ), cutLongNamesStep, transMeta ); + } + return steps.toArray( new StepMeta[ steps.size() ] ); } @@ -229,4 +248,148 @@ public Log getLogger() { return log; } + /** + * The target database maxColumnNameLength value if available; + * 0 otherwise. + * @return + */ + protected int getMaxColumnNameLength() { + int maxLen = 0; + Database db = null; + try { + db = this.getDatabase( getTargetDatabaseMeta() ); + if ( db == null ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): database is not available." ); //$NON-NLS-1$ + return maxLen; + } + db.connect( null ); + final DatabaseMetaData databaseMetaData = db.getDatabaseMetaData(); + if ( databaseMetaData == null ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): database metadata are not available." ); //$NON-NLS-1$ + return maxLen; + } + maxLen = databaseMetaData.getMaxColumnNameLength(); + } catch ( KettleDatabaseException e ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): " + e.getMessage(), e ); //$NON-NLS-1$ + } catch ( SQLException e ) { + log.debug( "Cannot getMaxColumnNameLength (defaults to 0): " + e.getMessage(), e ); //$NON-NLS-1$ + } finally { + if ( db != null ) { + db.disconnect(); + } + } + return maxLen; + } + + /** + * This step scans output fields of the last step in steps, + * + * cut field names that longer than maxColumnNameLength, + * + * renames them if necessary to keep them unique. + *
+ * If maxColumnNameLength<=0 or all field names are short enough, the step is not created; + * @param transMeta + * @param steps + * @param maxColumnNameLength + * @param stepName + * @return created {@link StepMeta} or null + * @throws KettleTransException + */ + protected StepMeta createCutLongNamesStep( TransMeta transMeta, List steps, int maxColumnNameLength, String stepName ) { + if ( maxColumnNameLength <= 0 ) { + return null; + } + try { + StepMeta prevStepMeta = steps.get( steps.size() - 1 ); + RowMetaInterface fields = transMeta.getStepFields( prevStepMeta ); + StepMeta stepMeta = createCutLongNamesStep( fields, maxColumnNameLength, stepName ); + if ( stepMeta != null ) { + transMeta.addStep( stepMeta ); + } + return stepMeta; + } catch ( KettleTransException e ) { + log.warn( "Unable to createCutLongNamesStep. Skipping it.", e ); + } catch ( RuntimeException e ) { + log.warn( "Unable to createCutLongNamesStep. Skipping it.", e ); + } + return null; + } + + /** + * + * @param fields + * @param maxColumnNameLength + * @param stepName + * @return + * @throws KettleTransException + */ + protected StepMeta createCutLongNamesStep( RowMetaInterface fields, int maxColumnNameLength, String stepName ) throws KettleTransException { + final int fieldsCount = fields.size(); + + SelectValuesMeta meta = new SelectValuesMeta(); + List selectNameList = new ArrayList( fieldsCount ); + List selectRenameList = new ArrayList( fieldsCount ); + List selectLengthList = new ArrayList( fieldsCount ); + List selectPrecisionList = new ArrayList( fieldsCount ); + final Collection controlNames = new HashSet(); + boolean renameRequired = false; + for ( ValueMetaInterface valueMeta : fields.getValueMetaList() ) { + final String oldName = valueMeta.getName(); + selectNameList.add( oldName ); + String newName = oldName; + if ( newName.length() > maxColumnNameLength ) { + renameRequired = true; + newName = newName.substring( 0, maxColumnNameLength ); + } + if ( controlNames.contains( newName.toLowerCase() ) ) { + renameRequired = true; + newName = null; + String candidateName = null; + final int maxAppendableSuffixLength = maxColumnNameLength - oldName.length(); + for ( int j = 1; newName == null && j < Integer.MAX_VALUE; j++ ) { + String suffix = "_" + j; + if ( suffix.length() > maxColumnNameLength ) { + throw new KettleTransException( "Cannot cut field name. Maximum suffix length is exceeded" ); //$NON-NLS-1$ + } + if ( suffix.length() <= maxAppendableSuffixLength ) { + candidateName = oldName + suffix; + } else { + candidateName = oldName.substring( 0, maxColumnNameLength - suffix.length() ) + suffix; + } + if ( !controlNames.contains( candidateName.toLowerCase() ) ) { + newName = candidateName; + } + } + if ( newName == null ) { // This is fantastic but... let it be + throw new KettleTransException( "Cannot cut field name. Maximum trials number is reached." ); //$NON-NLS-1$ + } + } + controlNames.add( newName.toLowerCase() ); + selectRenameList.add( newName ); + selectLengthList.add( valueMeta.getLength() ); + selectPrecisionList.add( valueMeta.getPrecision() ); + } + if ( !renameRequired ) { + return null; + } + String[] selectName = selectNameList.toArray( new String[ selectNameList.size() ] ); + meta.setSelectName( selectName ); + String[] selectRename = selectRenameList.toArray( new String[ selectRenameList.size() ] ); + meta.setSelectRename( selectRename ); + + int[] selectLength = new int[ selectLengthList.size() ]; + int[] selectPrecision = new int[ selectPrecisionList.size() ]; + for ( int i = 0; i < selectLength.length; i++ ) { + selectLength[ i ] = selectLengthList.get( i ); + } + for ( int i = 0; i < selectPrecision.length; i++ ) { + selectPrecision[ i ] = selectPrecisionList.get( i ); + } + meta.setSelectLength( selectLength ); + meta.setSelectPrecision( selectPrecision ); + + StepMeta stepMeta = new StepMeta( stepName, stepName, meta ); + return stepMeta; + } } diff --git a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java index 940b0e42f..44b37aca6 100644 --- a/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java +++ b/src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGenerator.java @@ -690,4 +690,8 @@ private boolean checkTableExists( String tableName ) throws CsvTransformGenerato db.disconnect(); } } + + DatabaseMeta getTargetDatabaseMeta() { + return targetDatabaseMeta; + } } diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java index 167b30c9f..940617bfa 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java @@ -17,10 +17,35 @@ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import java.io.File; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.List; + import org.apache.commons.lang.ArrayUtils; +import org.junit.Assert; import org.mockito.Mockito; import org.pentaho.di.core.database.Database; import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.exception.KettleStepException; +import org.pentaho.di.core.row.RowMeta; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; +import org.pentaho.di.core.row.value.ValueMetaBigNumber; +import org.pentaho.di.trans.TransMeta; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; import org.pentaho.metadata.model.concept.types.AggregationType; import org.pentaho.metadata.model.concept.types.DataType; import org.pentaho.platform.api.engine.IApplicationContext; @@ -37,12 +62,6 @@ import org.pentaho.platform.plugin.action.kettle.KettleSystemListener; import org.pentaho.test.platform.engine.core.BaseTest; -import java.io.File; -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.Statement; -import java.util.Date; - @SuppressWarnings( { "all" } ) public class CsvTransformGeneratorIT extends BaseTest { @@ -56,6 +75,21 @@ public class CsvTransformGeneratorIT extends BaseTest { private static final String SYSTEM_FOLDER = "/system"; //$NON-NLS-1$ + static class CutLongNamesStepInputContext { + String[] fieldNames = new String[] {"a", "b", "A_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", }; + int[] fieldLengths = new int[] { 5, 6, 10, 15, 12, 7, 4 }; + int[] fieldPrecisions = new int[] { 0, 0, 1, 1, 2, 0, 2 }; + RowMetaInterface fields = new RowMeta(); + { + for ( int i = 0; i < fieldNames.length; i++ ) { + final ValueMetaInterface valueMeta = new ValueMetaBigNumber( fieldNames[i] ); + valueMeta.setLength( fieldLengths[i] ); + valueMeta.setPrecision( fieldPrecisions[i] ); + fields.addValueMeta( valueMeta ); + } + } + } + public String getSolutionPath() { File file = new File( SOLUTION_PATH + PENTAHO_XML_PATH ); if ( file.exists() ) { @@ -196,6 +230,54 @@ public void testCreateTable() throws Exception { assertEquals( (long) 0, rowCount ); } + public void testCreateTable_longColumnNames() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( 8 ).when( gen ).getMaxColumnNameLength(); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + gen.createOrModifyTable( session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + } + + public void testCreateTable_littleMaxColumnNameLength() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( 1 ).when( gen ).getMaxColumnNameLength(); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + gen.createOrModifyTable( session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + } + /** * Given a name of an existing table to drop. *
@@ -416,7 +498,7 @@ public void testModifyEmptyTable_AddColumn() throws Exception { // generate the database table initially gen.createOrModifyTable( session ); - // now, lets update it by changing the model info slightly.. add a column + // now, lets update it by changing the model info slightly.. add a column addColumnToModel( info ); gen.createOrModifyTable( session ); @@ -440,7 +522,7 @@ public void testModifyEmptyTable_RemoveColumn() throws Exception { String removedColumn = info.getColumns()[ info.getColumns().length - 1 ].getId(); - // now, lets update it by changing the model info slightly.. add a column + // now, lets update it by changing the model info slightly.. add a column removeColumnFromModel( info ); gen.createOrModifyTable( session ); @@ -542,7 +624,7 @@ private int loadTable( CsvTransformGenerator gen, ModelInfo info, boolean trunca FileTransformStats stats = gen.getTransformStats(); - // wait until it it done + // wait until it it done while ( !stats.isRowsFinished() ) { Thread.sleep( 100 ); } @@ -727,4 +809,174 @@ public static ModelInfo createModel() { return info; } + + private static DatabaseMeta getJndiDatabaseMeta( String jndi ) { + DatabaseMeta databaseMeta = new DatabaseMeta(); + databaseMeta.setAccessType( DatabaseMeta.TYPE_ACCESS_JNDI ); + databaseMeta.setDBName( jndi ); + databaseMeta.setName( jndi ); + return databaseMeta; + } + + public void testGetMaxColumnNameLength() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final String H2_JNDI = "pentaho_staging_H2"; // test-res/solution1/system/simple-jndi/jdbc.properties + final int H2_MAX_COLUMN_NAME_LENGTH = 0; // org.h2.jdbc.JdbcDatabaseMetaData: h2-1.0.78.jar + + final String HYPERSONIC_JNDI = "pentaho_staging_Hypersonic"; // test-res/solution1/system/simple-jndi/jdbc.properties + final int HYPERSONIC_MAX_COLUMN_NAME_LENGTH = 128; // org.hsqldb.jdbc.JDBCDatabaseMetaData: hsqldb-2.3.2.jar + + final String INVALID_JNDI = "some_invalid_jndi"; + final int DEFAULT_MAX_COLUMN_NAME_LENGTH = 0; + + CsvTransformGenerator genH2 = new CsvTransformGenerator( info, getJndiDatabaseMeta( H2_JNDI ) ); + assertEquals( H2_MAX_COLUMN_NAME_LENGTH, genH2.getMaxColumnNameLength() ); + + CsvTransformGenerator genHsqldb = new CsvTransformGenerator( info, getJndiDatabaseMeta( HYPERSONIC_JNDI ) ); + assertEquals( HYPERSONIC_MAX_COLUMN_NAME_LENGTH, genHsqldb.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalid = new CsvTransformGenerator( info, getJndiDatabaseMeta( INVALID_JNDI ) ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalid.getMaxColumnNameLength() ); + } + + public void testGetMaxColumnNameLength_specialCases() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + DatabaseMeta dbMeta = getDatabaseMeta(); + + final int DEFAULT_MAX_COLUMN_NAME_LENGTH = 0; + + CsvTransformGenerator genNoDB = spy( new CsvTransformGenerator( info, dbMeta ) ); + doReturn( null ).when( genNoDB ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genNoDB.getMaxColumnNameLength() ); + + CsvTransformGenerator genNoMetadata = spy( new CsvTransformGenerator( info, dbMeta ) ); + Database dbNoMetadata = mock( Database.class ); + doReturn( null ).when( dbNoMetadata ).getDatabaseMetaData(); + doReturn( dbNoMetadata ).when( genNoDB ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genNoMetadata.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalidDb = spy( new CsvTransformGenerator( info, dbMeta ) ); + Database dbInvalid = mock( Database.class ); + doThrow( new KettleDatabaseException() ).when( dbInvalid ).connect(); + doReturn( dbInvalid ).when( genInvalidDb ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalidDb.getMaxColumnNameLength() ); + + CsvTransformGenerator genInvalidDbMetadata = spy( new CsvTransformGenerator( info, dbMeta ) ); + final DatabaseMetaData dbMetaDataInvalid = mock( DatabaseMetaData.class ); + Database dbInvalidMetadata = mock( Database.class ); + doThrow( new SQLException() ).when( dbMetaDataInvalid ).getMaxColumnNameLength(); + doReturn( dbMetaDataInvalid ).when( dbInvalidMetadata ).getDatabaseMetaData(); + doReturn( dbInvalidMetadata ).when( genInvalidDbMetadata ).getDatabase( dbMeta ); + assertEquals( DEFAULT_MAX_COLUMN_NAME_LENGTH, genInvalidDbMetadata.getMaxColumnNameLength() ); + + } + + public void testCreateCutLongNamesStep_longColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 8; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNotNull( "step", step ); + Assert.assertEquals( "step name", STEP_NAME, step.getName() ); + + StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); + Assert.assertNotNull( "stepMetaIntegrface", stepMetaIntegrface ); + Assert.assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); + SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; + + String[] selectName = svm.getSelectName(); + Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + String[] selectRename = svm.getSelectRename(); + Assert.assertArrayEquals( "selectRename", fieldRenames, selectRename ); + int[] selectLengths = svm.getSelectLength(); + Assert.assertArrayEquals( "selectLength", prev.fieldLengths, selectLengths ); + int[] selectPrecisions = svm.getSelectPrecision(); + Assert.assertArrayEquals( "selectPrecision", selectPrecisions, selectPrecisions ); + + Assert.assertEquals( step, transMeta.findStep( STEP_NAME ) ); + } + + public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNull( "step", step ); + } + + public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 1; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + assertNull( step ); + } + + public void testCreateCutLongNamesStep_prevStepError() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; + final String STEP_NAME = "TEST_STEP_CutLongNames"; + + StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required + List steps = java.util.Collections.singletonList( prevStepMeta ); + TransMeta transMeta = spy( new TransMeta() ); + doThrow( new KettleStepException() ).when( transMeta ).getStepFields( prevStepMeta ); + + final DatabaseMeta databaseMeta = getDatabaseMeta(); + CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); + + StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + + Assert.assertNull( "step", step ); + } + } diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java new file mode 100644 index 000000000..354dd6926 --- /dev/null +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java @@ -0,0 +1,78 @@ +/*! +* This program is free software; you can redistribute it and/or modify it under the +* terms of the GNU Lesser General Public License, version 2.1 as published by the Free Software +* Foundation. +* +* You should have received a copy of the GNU Lesser General Public License along with this +* program; if not, you can obtain a copy at http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +* or from the Free Software Foundation, Inc., +* 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +* +* This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; +* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +* See the GNU Lesser General Public License for more details. +* +* Copyright (c) 2002-2016 Pentaho Corporation.. All rights reserved. +*/ +package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; + +import static org.junit.Assert.fail; + +import org.junit.Assert; +import org.junit.Test; +import org.pentaho.di.trans.step.StepMeta; +import org.pentaho.di.trans.step.StepMetaInterface; +import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; +import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; +import org.pentaho.platform.dataaccess.datasource.wizard.service.agile.CsvTransformGeneratorIT.CutLongNamesStepInputContext; + +public class CsvTransformGeneratorTest { + + @Test + public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 18; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + Assert.assertNull( "step", step ); + } + + @Test + public void testCreateCutLongNamesStep_longColumnNames() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 8; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + Assert.assertNotNull( "step", step ); + Assert.assertEquals( "step name", stepName, step.getName() ); + StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); + Assert.assertNotNull( "stepMetaIntegrface", stepMetaIntegrface ); + Assert.assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); + SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; + String[] selectName = svm.getSelectName(); + Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + String[] selectRename = svm.getSelectRename(); + Assert.assertArrayEquals( "selectName", fieldRenames, selectRename ); + } + + @Test + public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 1; + String stepName = "TEST_STEP_CutLongNames"; + CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + + try { + StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + fail( "Ex[pected exception: Cannot cut field name. Maximum suffix length is exceeded" ); + } catch ( Exception e ) { + // expected + } + } + + +} From 2580955ace79d6afabeb59fb08d5f2ecd50edf59 Mon Sep 17 00:00:00 2001 From: PogI Date: Thu, 28 Apr 2016 15:30:03 +0400 Subject: [PATCH 2/3] [BACKLOG-6461] add tests --- .../agile/CsvTransformGeneratorIT.java | 153 +++++++++++++++--- .../agile/CsvTransformGeneratorTest.java | 18 +-- .../agile/StagingTransformGeneratorTest.java | 23 +++ 3 files changed, 163 insertions(+), 31 deletions(-) diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java index 940617bfa..c59483488 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java @@ -17,10 +17,15 @@ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; import java.io.File; import java.sql.Connection; @@ -75,10 +80,12 @@ public class CsvTransformGeneratorIT extends BaseTest { private static final String SYSTEM_FOLDER = "/system"; //$NON-NLS-1$ - static class CutLongNamesStepInputContext { - String[] fieldNames = new String[] {"a", "b", "A_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", }; - int[] fieldLengths = new int[] { 5, 6, 10, 15, 12, 7, 4 }; - int[] fieldPrecisions = new int[] { 0, 0, 1, 1, 2, 0, 2 }; + static class CutLongNamesStepContext { + String[] fieldNames = new String[] {"a", "b", "A_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", "lonGlo_1"}; + String[] fieldRenamesCut3 = new String[] {"a", "b", "A_1", "b_1", "LON", "l_1", "a_2", "l_2"}; + String[] fieldRenamesCut8 = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2", "lonGlo_2"}; + int[] fieldLengths = new int[] { 5, 6, 10, 15, 12, 7, 4, 4 }; + int[] fieldPrecisions = new int[] { 0, 0, 1, 1, 2, 0, 2, 2 }; RowMetaInterface fields = new RowMeta(); { for ( int i = 0; i < fieldNames.length; i++ ) { @@ -238,7 +245,8 @@ public void testCreateTable_longColumnNames() throws Exception { DatabaseMeta dbMeta = getDatabaseMeta(); ModelInfo info = createModel(); CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); - doReturn( 8 ).when( gen ).getMaxColumnNameLength(); + final int maxColumnNameLength = 3; + doReturn( maxColumnNameLength ).when( gen ).getMaxColumnNameLength(); String tableName = info.getStageTableName(); @@ -248,10 +256,12 @@ public void testCreateTable_longColumnNames() throws Exception { // it is OK if the table doesn't exist previously } gen.createOrModifyTable( session ); + verify( gen, atLeastOnce() ).createCutLongNamesStep( any( TransMeta.class ), any( List.class ), eq( maxColumnNameLength ), anyString() ); // check the results long rowCount = this.getRowCount( tableName ); assertEquals( (long) 0, rowCount ); + assertEquals( 0, gen.getTransformStats().getErrorCount() ); } public void testCreateTable_littleMaxColumnNameLength() throws Exception { @@ -262,7 +272,8 @@ public void testCreateTable_littleMaxColumnNameLength() throws Exception { DatabaseMeta dbMeta = getDatabaseMeta(); ModelInfo info = createModel(); CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); - doReturn( 1 ).when( gen ).getMaxColumnNameLength(); + final int maxColumnNameLength = 1; + doReturn( maxColumnNameLength ).when( gen ).getMaxColumnNameLength(); String tableName = info.getStageTableName(); @@ -272,6 +283,7 @@ public void testCreateTable_littleMaxColumnNameLength() throws Exception { // it is OK if the table doesn't exist previously } gen.createOrModifyTable( session ); + verify( gen, atLeastOnce() ).createCutLongNamesStep( any( TransMeta.class ), any( List.class ), eq( maxColumnNameLength ), anyString() ); // check the results long rowCount = this.getRowCount( tableName ); @@ -442,6 +454,102 @@ public void testLoadTable1() throws Exception { assertTrue( ok ); sqlResult = stmt.getResultSet(); assertNotNull( sqlResult ); + assertEquals( 1, sqlResult.findColumn( "PC_0" ) ); + assertEquals( 2, sqlResult.findColumn( "PC_1" ) ); + assertEquals( 3, sqlResult.findColumn( "PC_2" ) ); + assertEquals( 4, sqlResult.findColumn( "PC_3" ) ); + assertEquals( 5, sqlResult.findColumn( "PC_5" ) ); + assertEquals( 6, sqlResult.findColumn( "PC_6" ) ); + assertEquals( 7, sqlResult.findColumn( "PC_7" ) ); + assertEquals( 8, sqlResult.findColumn( "PC_8" ) ); + ok = sqlResult.next(); + assertTrue( ok ); + + // test the values + assertEquals( (long) 3, sqlResult.getLong( 1 ) ); + assertEquals( 25677.96525, sqlResult.getDouble( 2 ) ); + assertEquals( (long) 1231, sqlResult.getLong( 3 ) ); + assertEquals( testDate.getYear(), sqlResult.getDate( 4 ).getYear() ); + assertEquals( testDate.getMonth(), sqlResult.getDate( 4 ).getMonth() ); + assertEquals( testDate.getDate(), sqlResult.getDate( 4 ).getDate() ); + assertEquals( testDate.getHours(), sqlResult.getTime( 4 ).getHours() ); + // assertEquals( testDate.getMinutes(), ((Date)cells[3]).getMinutes() ); this fails, a bug in the PDI date + // parsing? + assertEquals( testDate.getSeconds(), sqlResult.getTime( 4 ).getSeconds() ); + + // assertEquals( testDate, cells[3] ); + assertEquals( "Afghanistan", sqlResult.getString( 5 ) ); + assertEquals( (long) 11, sqlResult.getLong( 6 ) ); + assertEquals( 111.9090909, sqlResult.getDouble( 7 ) ); + assertEquals( false, sqlResult.getBoolean( 8 ) ); + } finally { + sqlResult.close(); + stmt.close(); + connection.close(); + } + + } + + public void testLoadTable_longColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, getDatabaseMeta() ) ); + final int maxColumnNameLength = 3; + doReturn( maxColumnNameLength ).when( gen ).getMaxColumnNameLength(); + + // create the model + String tableName = info.getStageTableName(); + try { + gen.execSqlStatement( getDropTableStatement( tableName ), getDatabaseMeta(), null ); + } catch ( CsvTransformGeneratorException e ) { + // table might not be there yet, it is OK + } + + // generate the database table + gen.createOrModifyTable( session ); + + // load the table + loadTable( gen, info, true, session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 235, rowCount ); + DatabaseMeta databaseMeta = getDatabaseMeta(); + assertNotNull( databaseMeta ); + Database database = new Database( databaseMeta ); + assertNotNull( database ); + database.connect(); + + Connection connection = null; + Statement stmt = null; + ResultSet sqlResult = null; + + try { + connection = database.getConnection(); + assertNotNull( connection ); + stmt = database.getConnection().createStatement(); + + // check the first row + Date testDate = new Date(); + testDate.setDate( 1 ); + testDate.setHours( 0 ); + testDate.setMinutes( 0 ); + testDate.setMonth( 0 ); + testDate.setSeconds( 0 ); + testDate.setYear( 110 ); + boolean ok = stmt.execute( "select * from " + tableName ); + assertTrue( ok ); + sqlResult = stmt.getResultSet(); + assertNotNull( sqlResult ); + assertEquals( 1, sqlResult.findColumn( "PC_" ) ); // PC_0 cut + assertEquals( 2, sqlResult.findColumn( "P_1" ) ); // PC_1 cut unique + assertEquals( 3, sqlResult.findColumn( "P_2" ) ); // PC_2 cut unique + assertEquals( 4, sqlResult.findColumn( "P_3" ) ); // PC_3 cut unique + assertEquals( 5, sqlResult.findColumn( "P_4" ) ); // PC_5 cut unique + assertEquals( 6, sqlResult.findColumn( "P_5" ) ); // PC_6 cut unique + assertEquals( 7, sqlResult.findColumn( "P_6" ) ); // PC_7 cut unique + assertEquals( 8, sqlResult.findColumn( "P_7" ) ); // PC_8 cut unique ok = sqlResult.next(); assertTrue( ok ); @@ -877,18 +985,19 @@ public void testGetMaxColumnNameLength_specialCases() throws Exception { } public void testCreateCutLongNamesStep_longColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); KettleSystemListener.environmentInit( session ); ModelInfo info = createModel(); - final int DATABASE_MAX_COLUMN_NAME_LENGTH = 8; final String STEP_NAME = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); - String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 8; + final String[] FIELD_RENAMES = testData.fieldRenamesCut8; StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required List steps = java.util.Collections.singletonList( prevStepMeta ); TransMeta transMeta = spy( new TransMeta() ); - doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + doReturn( testData.fields ).when( transMeta ).getStepFields( prevStepMeta ); final DatabaseMeta databaseMeta = getDatabaseMeta(); CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); @@ -904,11 +1013,11 @@ public void testCreateCutLongNamesStep_longColumnNames() throws Exception { SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; String[] selectName = svm.getSelectName(); - Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + Assert.assertArrayEquals( "selectName", testData.fieldNames, selectName ); String[] selectRename = svm.getSelectRename(); - Assert.assertArrayEquals( "selectRename", fieldRenames, selectRename ); + Assert.assertArrayEquals( "selectRename", FIELD_RENAMES, selectRename ); int[] selectLengths = svm.getSelectLength(); - Assert.assertArrayEquals( "selectLength", prev.fieldLengths, selectLengths ); + Assert.assertArrayEquals( "selectLength", testData.fieldLengths, selectLengths ); int[] selectPrecisions = svm.getSelectPrecision(); Assert.assertArrayEquals( "selectPrecision", selectPrecisions, selectPrecisions ); @@ -916,18 +1025,18 @@ public void testCreateCutLongNamesStep_longColumnNames() throws Exception { } public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); KettleSystemListener.environmentInit( session ); ModelInfo info = createModel(); - final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; final String STEP_NAME = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); - + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required List steps = java.util.Collections.singletonList( prevStepMeta ); TransMeta transMeta = spy( new TransMeta() ); - doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + doReturn( testData.fields ).when( transMeta ).getStepFields( prevStepMeta ); final DatabaseMeta databaseMeta = getDatabaseMeta(); CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); @@ -942,19 +1051,19 @@ public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Except KettleSystemListener.environmentInit( session ); ModelInfo info = createModel(); - final int DATABASE_MAX_COLUMN_NAME_LENGTH = 1; final String STEP_NAME = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); - + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 1; StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required List steps = java.util.Collections.singletonList( prevStepMeta ); TransMeta transMeta = spy( new TransMeta() ); - doReturn( prev.fields ).when( transMeta ).getStepFields( prevStepMeta ); + doReturn( testData.fields ).when( transMeta ).getStepFields( prevStepMeta ); final DatabaseMeta databaseMeta = getDatabaseMeta(); CsvTransformGenerator gen = new CsvTransformGenerator( info, databaseMeta ); StepMeta step = gen.createCutLongNamesStep( transMeta, steps, DATABASE_MAX_COLUMN_NAME_LENGTH, STEP_NAME ); + assertNull( step ); } @@ -963,8 +1072,8 @@ public void testCreateCutLongNamesStep_prevStepError() throws Exception { KettleSystemListener.environmentInit( session ); ModelInfo info = createModel(); - final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; final String STEP_NAME = "TEST_STEP_CutLongNames"; + final int DATABASE_MAX_COLUMN_NAME_LENGTH = 18; StepMeta prevStepMeta = mock( StepMeta.class ); // No functionality is required List steps = java.util.Collections.singletonList( prevStepMeta ); diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java index 354dd6926..1486c2576 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java @@ -24,7 +24,7 @@ import org.pentaho.di.trans.step.StepMetaInterface; import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; -import org.pentaho.platform.dataaccess.datasource.wizard.service.agile.CsvTransformGeneratorIT.CutLongNamesStepInputContext; +import org.pentaho.platform.dataaccess.datasource.wizard.service.agile.CsvTransformGeneratorIT.CutLongNamesStepContext; public class CsvTransformGeneratorTest { @@ -33,8 +33,8 @@ public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); int maxColumnNameLength = 18; String stepName = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); - StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + StepMeta step = ctg.createCutLongNamesStep( testData.fields, maxColumnNameLength, stepName ); Assert.assertNull( "step", step ); } @@ -43,10 +43,10 @@ public void testCreateCutLongNamesStep_longColumnNames() throws Exception { CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); int maxColumnNameLength = 8; String stepName = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); - String[] fieldRenames = new String[] {"a", "b", "A_1", "b_1", "LONGlong", "longlo_1", "a_2"}; + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + String[] fieldRenames = testData.fieldRenamesCut8; - StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + StepMeta step = ctg.createCutLongNamesStep( testData.fields, maxColumnNameLength, stepName ); Assert.assertNotNull( "step", step ); Assert.assertEquals( "step name", stepName, step.getName() ); StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); @@ -54,7 +54,7 @@ public void testCreateCutLongNamesStep_longColumnNames() throws Exception { Assert.assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; String[] selectName = svm.getSelectName(); - Assert.assertArrayEquals( "selectName", prev.fieldNames, selectName ); + Assert.assertArrayEquals( "selectName", testData.fieldNames, selectName ); String[] selectRename = svm.getSelectRename(); Assert.assertArrayEquals( "selectName", fieldRenames, selectRename ); } @@ -64,10 +64,10 @@ public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Except CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); int maxColumnNameLength = 1; String stepName = "TEST_STEP_CutLongNames"; - CutLongNamesStepInputContext prev = new CutLongNamesStepInputContext(); + CutLongNamesStepContext testData = new CutLongNamesStepContext(); try { - StepMeta step = ctg.createCutLongNamesStep( prev.fields, maxColumnNameLength, stepName ); + StepMeta step = ctg.createCutLongNamesStep( testData.fields, maxColumnNameLength, stepName ); fail( "Ex[pected exception: Cannot cut field name. Maximum suffix length is exceeded" ); } catch ( Exception e ) { // expected diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGeneratorTest.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGeneratorTest.java index ab8ea51cd..e792a3303 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGeneratorTest.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/StagingTransformGeneratorTest.java @@ -25,11 +25,16 @@ import org.pentaho.di.core.plugins.DatabasePluginType; import org.pentaho.platform.dataaccess.datasource.wizard.models.ModelInfo; +import static org.junit.Assert.assertEquals; +import static org.mockito.Matchers.any; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; import static org.mockito.Matchers.isNull; +import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -93,4 +98,22 @@ public void shouldNotDropTableIfNotExists() throws Exception { verify( database, never() ).execStatement( anyString() ); } + + @Test + public void testGetTargetDatabaseMeta() throws Exception { + assertEquals( databaseMeta, stagingTransformGenerator.getTargetDatabaseMeta() ); + } + + @Test + public void testDropTable() throws Exception { + final String tableName = "tableName"; + final String quotedTableName = "\"" + tableName + "\""; + final String expectedDdl = "DROP TABLE " + quotedTableName; + + doReturn( quotedTableName ).when( databaseMeta ).getQuotedSchemaTableCombination( anyString(), eq( tableName ) ); + doReturn( true ).when( database ).checkTableExists( quotedTableName ); + StagingTransformGenerator stg = spy( stagingTransformGenerator ); + stg.dropTable( tableName ); + verify( stg, times( 1 ) ).execSqlStatement( eq( expectedDdl ), eq( databaseMeta ), any( StringBuilder.class ) ); + } } From 5631d0972de814b0b5e26712ffc32650dd4482c2 Mon Sep 17 00:00:00 2001 From: PogI Date: Fri, 29 Apr 2016 15:26:22 +0400 Subject: [PATCH 3/3] [BACKLOG-6461] more tests --- .../agile/CsvTransformGeneratorIT.java | 107 ++++++++++++++++++ .../agile/CsvTransformGeneratorTest.java | 95 +++++++++++++++- 2 files changed, 201 insertions(+), 1 deletion(-) diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java index c59483488..cac7aa5d2 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorIT.java @@ -214,6 +214,15 @@ public void testGoodTransform() throws Exception { } + public void testInit_DefaultDb() throws Exception { + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + ModelInfo info = createModel(); + CsvTransformGenerator gen = new CsvTransformGenerator( info ); + assertEquals( getDatabaseMeta(), gen.getTargetDatabaseMeta() ); + } + public void testCreateTable() throws Exception { IPentahoSession session = new StandaloneSession( "test" ); @@ -237,6 +246,67 @@ public void testCreateTable() throws Exception { assertEquals( (long) 0, rowCount ); } + public void testCreateTable_noIgnoredColumns() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + ColumnInfo[] columns = info.getColumns(); + columns[ 4 ].setIgnore( false ); + info.setColumns( columns ); + CsvTransformGenerator gen = new CsvTransformGenerator( info, dbMeta ); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + gen.createOrModifyTable( session ); + + // check the results + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + + + DatabaseMeta databaseMeta = getDatabaseMeta(); + assertNotNull( databaseMeta ); + Database database = new Database( databaseMeta ); + assertNotNull( database ); + database.connect(); + + Connection connection = null; + Statement stmt = null; + ResultSet sqlResult = null; + + try { + connection = database.getConnection(); + assertNotNull( connection ); + stmt = database.getConnection().createStatement(); + + boolean ok = stmt.execute( "select * from " + tableName ); + assertTrue( ok ); + sqlResult = stmt.getResultSet(); + assertNotNull( sqlResult ); + assertEquals( 1, sqlResult.findColumn( "PC_0" ) ); + assertEquals( 2, sqlResult.findColumn( "PC_1" ) ); + assertEquals( 3, sqlResult.findColumn( "PC_2" ) ); + assertEquals( 4, sqlResult.findColumn( "PC_3" ) ); + assertEquals( 5, sqlResult.findColumn( "PC_4" ) ); + assertEquals( 6, sqlResult.findColumn( "PC_5" ) ); + assertEquals( 7, sqlResult.findColumn( "PC_6" ) ); + assertEquals( 8, sqlResult.findColumn( "PC_7" ) ); + assertEquals( 9, sqlResult.findColumn( "PC_8" ) ); + } finally { + sqlResult.close(); + stmt.close(); + connection.close(); + } + } + public void testCreateTable_longColumnNames() throws Exception { IPentahoSession session = new StandaloneSession( "test" ); @@ -264,6 +334,43 @@ public void testCreateTable_longColumnNames() throws Exception { assertEquals( 0, gen.getTransformStats().getErrorCount() ); } + public void testCreateTable_CannotCreateCutLongNamesStep() throws Exception { + + IPentahoSession session = new StandaloneSession( "test" ); + KettleSystemListener.environmentInit( session ); + + DatabaseMeta dbMeta = getDatabaseMeta(); + ModelInfo info = createModel(); + CsvTransformGenerator gen = spy( new CsvTransformGenerator( info, dbMeta ) ); + final int maxColumnNameLength = 3; + doReturn( maxColumnNameLength ).when( gen ).getMaxColumnNameLength(); + + String tableName = info.getStageTableName(); + + try { + gen.execSqlStatement( getDropTableStatement( tableName ), dbMeta, null ); + } catch ( CsvTransformGeneratorException e ) { + // it is OK if the table doesn't exist previously + } + try { + this.getRowCount( tableName ); + fail( "Test table must not exist now" ); + } catch ( SQLException e ) { + // ok + } + + doThrow( new RuntimeException() ).when( gen ).createCutLongNamesStep( any( RowMetaInterface.class ), eq( maxColumnNameLength ), anyString() ); + // This error is not critical. It's supposed to be skipped. + + gen.createOrModifyTable( session ); + verify( gen, atLeastOnce() ).createCutLongNamesStep( any( TransMeta.class ), any( List.class ), eq( maxColumnNameLength ), anyString() ); + + // check the results. + long rowCount = this.getRowCount( tableName ); + assertEquals( (long) 0, rowCount ); + assertEquals( 0, gen.getTransformStats().getErrorCount() ); + } + public void testCreateTable_littleMaxColumnNameLength() throws Exception { IPentahoSession session = new StandaloneSession( "test" ); diff --git a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java index 1486c2576..c47f73da7 100644 --- a/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java +++ b/test-src/org/pentaho/platform/dataaccess/datasource/wizard/service/agile/CsvTransformGeneratorTest.java @@ -16,10 +16,29 @@ */ package org.pentaho.platform.dataaccess.datasource.wizard.service.agile; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; + +import java.sql.DatabaseMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; import org.junit.Assert; import org.junit.Test; +import org.pentaho.di.core.database.Database; +import org.pentaho.di.core.database.DatabaseMeta; +import org.pentaho.di.core.exception.KettleDatabaseException; +import org.pentaho.di.core.row.RowMetaInterface; +import org.pentaho.di.core.row.ValueMetaInterface; import org.pentaho.di.trans.step.StepMeta; import org.pentaho.di.trans.step.StepMetaInterface; import org.pentaho.di.trans.steps.selectvalues.SelectValuesMeta; @@ -28,6 +47,49 @@ public class CsvTransformGeneratorTest { + @Test + public void testGetMaxColumnNameLength() throws Exception { + + CsvTransformGenerator ctg10 = spy( new CsvTransformGenerator( new ModelInfo(), null ) ); + Database db10 = mock( Database.class ); + doReturn( db10 ).when( ctg10 ).getDatabase( any( DatabaseMeta.class ) ); + DatabaseMetaData dbmd10 = mock( DatabaseMetaData.class ); + doReturn( dbmd10 ).when( db10 ).getDatabaseMetaData(); + doReturn( 10 ).when( dbmd10 ).getMaxColumnNameLength(); + assertEquals( 10, ctg10.getMaxColumnNameLength() ); + + CsvTransformGenerator ctg0 = spy( new CsvTransformGenerator( new ModelInfo(), null ) ); + Database db0 = mock( Database.class ); + doReturn( db0 ).when( ctg0 ).getDatabase( any( DatabaseMeta.class ) ); + DatabaseMetaData dbmd0 = mock( DatabaseMetaData.class ); + doReturn( dbmd0 ).when( db0 ).getDatabaseMetaData(); + doReturn( 0 ).when( dbmd0 ).getMaxColumnNameLength(); + assertEquals( 0, ctg0.getMaxColumnNameLength() ); + + // As for the case of doReturn( ).when( dbmd0 ).getMaxColumnNameLength(); + // It's impossible according to java.sql.DatabaseMetaData. + // @see java.sql.DatabaseMetaData.getMaxColumnNameLength() + // + + CsvTransformGenerator ctgErr1 = spy( new CsvTransformGenerator( new ModelInfo(), null ) ); + Database dbErr1 = mock( Database.class ); + doReturn( dbErr1 ).when( ctgErr1 ).getDatabase( any( DatabaseMeta.class ) ); + DatabaseMetaData dbmdErr1 = mock( DatabaseMetaData.class ); + doReturn( dbmdErr1 ).when( dbErr1 ).getDatabaseMetaData(); + doThrow( new SQLException() ).when( dbmdErr1 ).getMaxColumnNameLength(); + assertEquals( 0, ctgErr1.getMaxColumnNameLength() ); + + CsvTransformGenerator ctgErr2 = spy( new CsvTransformGenerator( new ModelInfo(), null ) ); + Database dbErr2 = mock( Database.class ); + doReturn( dbErr2 ).when( ctgErr2 ).getDatabase( any( DatabaseMeta.class ) ); + doThrow( new KettleDatabaseException() ).when( dbErr2 ).getDatabaseMetaData(); + assertEquals( 0, ctgErr2.getMaxColumnNameLength() ); + + CsvTransformGenerator ctgNoDatabase = spy( new CsvTransformGenerator( new ModelInfo(), null ) ); + doReturn( null ).when( ctgNoDatabase ).getDatabase( any( DatabaseMeta.class ) ); + assertEquals( 0, ctgNoDatabase.getMaxColumnNameLength() ); + } + @Test public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); @@ -38,6 +100,38 @@ public void testCreateCutLongNamesStep_shortColumnNames() throws Exception { Assert.assertNull( "step", step ); } + @Test + /** + * Very artificial case. + * org.pentaho.di.core.row.RowMeta provides unique names, + * though it's not required in org.pentaho.di.core.row.RowMetaInterface + */ + public void testCreateCutLongNamesStep_dupColumnNames() throws Exception { + CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); + int maxColumnNameLength = 18; + String stepName = "TEST_STEP_DupLongNames"; + CutLongNamesStepContext testData = new CutLongNamesStepContext(); + RowMetaInterface fields = spy( testData.fields ); + List vmList = new ArrayList( testData.fields.getValueMetaList() ); + vmList.set( 1, vmList.get( 0 ) ); + doReturn( vmList ).when( fields ).getValueMetaList(); + + String[] fieldNamesDups = new String[] {"a", "a", "A_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", "lonGlo_1"}; + String[] fieldRenames = new String[] {"a", "a_1", "A_1_1", "b_1", "LONGlonglong", "longlonglong_again", "a_2", "lonGlo_1"}; + + StepMeta step = ctg.createCutLongNamesStep( fields, maxColumnNameLength, stepName ); + assertNotNull( "step", step ); + assertEquals( "step name", stepName, step.getName() ); + StepMetaInterface stepMetaIntegrface = step.getStepMetaInterface(); + assertNotNull( "stepMetaIntegrface", stepMetaIntegrface ); + assertTrue( "stepMetaIntegrface instanceof SelectValuesMeta", stepMetaIntegrface instanceof SelectValuesMeta ); + SelectValuesMeta svm = (SelectValuesMeta) stepMetaIntegrface; + String[] selectName = svm.getSelectName(); + assertArrayEquals( "selectName", fieldNamesDups, selectName ); + String[] selectRename = svm.getSelectRename(); + assertArrayEquals( "selectName", fieldRenames, selectRename ); + } + @Test public void testCreateCutLongNamesStep_longColumnNames() throws Exception { CsvTransformGenerator ctg = new CsvTransformGenerator( new ModelInfo(), null ); @@ -74,5 +168,4 @@ public void testCreateCutLongNamesStep_littleMaxColumnNameLength() throws Except } } - }