From a3d950f80ba026f72ba0014287fd5afd8528c8bf Mon Sep 17 00:00:00 2001 From: Ivy Zheng Date: Mon, 5 Jul 2021 19:47:17 +1000 Subject: [PATCH 1/4] Adjust NVivo 12 Adjust NVivo 12 (using MSSQL14) --- AdjustDate.py | 0 CompareDBs.py | 0 DeleteAllData.py | 0 DenormaliseDB.py | 0 DropItemForeignKeys.py | 0 NVPX2RQDA.py | 0 NVivo database structure notes.txt | 0 NVivo.py | 6 ++++-- NVivo2RQDADB.py | 0 Norm2RQDA.py | 0 NormaliseDB.py | 0 NormaliseNVP.py | 14 ++++++++++---- NormaliseNVPX.py | 0 NormaliseOpenQDA.py | 0 RQDA2NVPX.py | 0 RQDA2NVivoDB.py | 0 RQDA2Norm.py | 0 Subtract.py | 0 Translate.py | 0 editNode.py | 0 editNodeAttribute.py | 0 editNodeCategory.py | 0 editProject.py | 0 editSource.py | 0 editSourceAttribute.py | 0 editSourceCategory.py | 0 editTagging.py | 0 editUser.py | 0 extractTagging.py | 0 helpers/mssqlAttach.bat | 0 helpers/mssqlCreate.bat | 0 helpers/mssqlSave.bat | 0 helpers/sqlanyenv.sh | 0 helpers/sqlanysrv.sh | 0 helpers/unoconv | 0 mssqlAttach.py | 0 mssqlCreate.py | 0 mssqlDrop.py | 0 mssqlList.py | 0 mssqlSave.py | 0 nvpn2bqda.py | 0 nvpn2nvp.py | 0 nvpn2nvpx.py | 0 querySource.py | 0 queryTagging.py | 0 regressionTest.sh | 0 saveSources.py | 0 tagNounPhrases.py | 0 tagSpeakers.py | 0 49 files changed, 14 insertions(+), 6 deletions(-) mode change 100755 => 100644 AdjustDate.py mode change 100755 => 100644 CompareDBs.py mode change 100755 => 100644 DeleteAllData.py mode change 100755 => 100644 DenormaliseDB.py mode change 100755 => 100644 DropItemForeignKeys.py mode change 100755 => 100644 NVPX2RQDA.py mode change 100755 => 100644 NVivo database structure notes.txt mode change 100755 => 100644 NVivo2RQDADB.py mode change 100755 => 100644 Norm2RQDA.py mode change 100755 => 100644 NormaliseDB.py mode change 100755 => 100644 NormaliseNVP.py mode change 100755 => 100644 NormaliseNVPX.py mode change 100755 => 100644 NormaliseOpenQDA.py mode change 100755 => 100644 RQDA2NVPX.py mode change 100755 => 100644 RQDA2NVivoDB.py mode change 100755 => 100644 RQDA2Norm.py mode change 100755 => 100644 Subtract.py mode change 100755 => 100644 Translate.py mode change 100755 => 100644 editNode.py mode change 100755 => 100644 editNodeAttribute.py mode change 100755 => 100644 editNodeCategory.py mode change 100755 => 100644 editProject.py mode change 100755 => 100644 editSource.py mode change 100755 => 100644 editSourceAttribute.py mode change 100755 => 100644 editSourceCategory.py mode change 100755 => 100644 editTagging.py mode change 100755 => 100644 editUser.py mode change 100755 => 100644 extractTagging.py mode change 100755 => 100644 helpers/mssqlAttach.bat mode change 100755 => 100644 helpers/mssqlCreate.bat mode change 100755 => 100644 helpers/mssqlSave.bat mode change 100755 => 100644 helpers/sqlanyenv.sh mode change 100755 => 100644 helpers/sqlanysrv.sh mode change 100755 => 100644 helpers/unoconv mode change 100755 => 100644 mssqlAttach.py mode change 100755 => 100644 mssqlCreate.py mode change 100755 => 100644 mssqlDrop.py mode change 100755 => 100644 mssqlList.py mode change 100755 => 100644 mssqlSave.py mode change 100755 => 100644 nvpn2bqda.py mode change 100755 => 100644 nvpn2nvp.py mode change 100755 => 100644 nvpn2nvpx.py mode change 100755 => 100644 querySource.py mode change 100755 => 100644 queryTagging.py mode change 100755 => 100644 regressionTest.sh mode change 100755 => 100644 saveSources.py mode change 100755 => 100644 tagNounPhrases.py mode change 100755 => 100644 tagSpeakers.py diff --git a/AdjustDate.py b/AdjustDate.py old mode 100755 new mode 100644 diff --git a/CompareDBs.py b/CompareDBs.py old mode 100755 new mode 100644 diff --git a/DeleteAllData.py b/DeleteAllData.py old mode 100755 new mode 100644 diff --git a/DenormaliseDB.py b/DenormaliseDB.py old mode 100755 new mode 100644 diff --git a/DropItemForeignKeys.py b/DropItemForeignKeys.py old mode 100755 new mode 100644 diff --git a/NVPX2RQDA.py b/NVPX2RQDA.py old mode 100755 new mode 100644 diff --git a/NVivo database structure notes.txt b/NVivo database structure notes.txt old mode 100755 new mode 100644 diff --git a/NVivo.py b/NVivo.py index 235f647..1db1d6c 100644 --- a/NVivo.py +++ b/NVivo.py @@ -661,8 +661,10 @@ def Normalise(args): nvivoBlobStorage.c.Object if nvivoBlobStorage is not None else nvivoSource.c.Object, nvivoSource.c.PlainText, nvivoSource.c.MetaData, - nvivoSource.c.ThumbnailLocation if args.nvivoversion == '12' else nvivoSource.c.Thumbnail, - nvivoSource.c.WaveformLocation if args.nvivoversion == '12' else nvivoSource.c.Waveform, + # nvivoSource.c.ThumbnailLocation if args.nvivoversion in '12' else nvivoSource.c.Thumbnail, + nvivoSource.c.Thumbnail, + # nvivoSource.c.WaveformLocation if args.nvivoversion == '12' else nvivoSource.c.Waveform, + nvivoSource.c.Waveform, nvivoItem.c.TypeId.label('SourceType'), nvivoItem.c.CreatedBy, nvivoItem.c.CreatedDate, diff --git a/NVivo2RQDADB.py b/NVivo2RQDADB.py old mode 100755 new mode 100644 diff --git a/Norm2RQDA.py b/Norm2RQDA.py old mode 100755 new mode 100644 diff --git a/NormaliseDB.py b/NormaliseDB.py old mode 100755 new mode 100644 diff --git a/NormaliseNVP.py b/NormaliseNVP.py old mode 100755 new mode 100644 index d3b4c52..5aeeaf3 --- a/NormaliseNVP.py +++ b/NormaliseNVP.py @@ -31,8 +31,8 @@ def NormaliseNVP(arglist): parser.add_argument('-v', '--verbosity', type=int, default=1) - parser.add_argument('-nv', '--nvivoversion', choices=["10", "11"], default="10", - help='NVivo version (10 or 11)') + parser.add_argument('-nv', '--nvivoversion', choices=["10", "11", "12"], default="10", + help='NVivo version (10 or 11 or 12)') parser.add_argument('-S', '--server', type=str, help="IP address/name of Microsoft SQL Server") @@ -98,7 +98,7 @@ def executecommand(command): instanceversion = regquerydata[2].split('.')[0] if args.verbosity >= 2: print("Found SQL server instance " + instancename + " version " + instanceversion, file=sys.stderr) - if (args.nvivoversion == '10' and instanceversion == 'MSSQL10_50') or (args.nvivoversion == '11' and instanceversion == 'MSSQL12'): + if (args.nvivoversion == '10' and instanceversion == 'MSSQL10_50') or (args.nvivoversion == '11' and instanceversion == 'MSSQL12') or (args.nvivoversion == '12' and instanceversion == 'MSSQL14'): args.instance = instancename break else: @@ -114,10 +114,16 @@ def executecommand(command): if args.verbosity > 0: print("Using port: " + str(args.port), file=sys.stderr) + if args.nvivoversion == '11': + version = 'MSSQL12' + elif args.nvivoversion == '12': + version = 'MSSQL14' + else: + version = 'MSQL10_50' mssqlapi = mssqlAPI(args.server, args.port, args.instance, - version = ('MSSQL12' if args.nvivoversion == '11' else 'MSSQL10_50'), + version = version, verbosity = args.verbosity) # Get reasonably distinct yet recognisable DB name diff --git a/NormaliseNVPX.py b/NormaliseNVPX.py old mode 100755 new mode 100644 diff --git a/NormaliseOpenQDA.py b/NormaliseOpenQDA.py old mode 100755 new mode 100644 diff --git a/RQDA2NVPX.py b/RQDA2NVPX.py old mode 100755 new mode 100644 diff --git a/RQDA2NVivoDB.py b/RQDA2NVivoDB.py old mode 100755 new mode 100644 diff --git a/RQDA2Norm.py b/RQDA2Norm.py old mode 100755 new mode 100644 diff --git a/Subtract.py b/Subtract.py old mode 100755 new mode 100644 diff --git a/Translate.py b/Translate.py old mode 100755 new mode 100644 diff --git a/editNode.py b/editNode.py old mode 100755 new mode 100644 diff --git a/editNodeAttribute.py b/editNodeAttribute.py old mode 100755 new mode 100644 diff --git a/editNodeCategory.py b/editNodeCategory.py old mode 100755 new mode 100644 diff --git a/editProject.py b/editProject.py old mode 100755 new mode 100644 diff --git a/editSource.py b/editSource.py old mode 100755 new mode 100644 diff --git a/editSourceAttribute.py b/editSourceAttribute.py old mode 100755 new mode 100644 diff --git a/editSourceCategory.py b/editSourceCategory.py old mode 100755 new mode 100644 diff --git a/editTagging.py b/editTagging.py old mode 100755 new mode 100644 diff --git a/editUser.py b/editUser.py old mode 100755 new mode 100644 diff --git a/extractTagging.py b/extractTagging.py old mode 100755 new mode 100644 diff --git a/helpers/mssqlAttach.bat b/helpers/mssqlAttach.bat old mode 100755 new mode 100644 diff --git a/helpers/mssqlCreate.bat b/helpers/mssqlCreate.bat old mode 100755 new mode 100644 diff --git a/helpers/mssqlSave.bat b/helpers/mssqlSave.bat old mode 100755 new mode 100644 diff --git a/helpers/sqlanyenv.sh b/helpers/sqlanyenv.sh old mode 100755 new mode 100644 diff --git a/helpers/sqlanysrv.sh b/helpers/sqlanysrv.sh old mode 100755 new mode 100644 diff --git a/helpers/unoconv b/helpers/unoconv old mode 100755 new mode 100644 diff --git a/mssqlAttach.py b/mssqlAttach.py old mode 100755 new mode 100644 diff --git a/mssqlCreate.py b/mssqlCreate.py old mode 100755 new mode 100644 diff --git a/mssqlDrop.py b/mssqlDrop.py old mode 100755 new mode 100644 diff --git a/mssqlList.py b/mssqlList.py old mode 100755 new mode 100644 diff --git a/mssqlSave.py b/mssqlSave.py old mode 100755 new mode 100644 diff --git a/nvpn2bqda.py b/nvpn2bqda.py old mode 100755 new mode 100644 diff --git a/nvpn2nvp.py b/nvpn2nvp.py old mode 100755 new mode 100644 diff --git a/nvpn2nvpx.py b/nvpn2nvpx.py old mode 100755 new mode 100644 diff --git a/querySource.py b/querySource.py old mode 100755 new mode 100644 diff --git a/queryTagging.py b/queryTagging.py old mode 100755 new mode 100644 diff --git a/regressionTest.sh b/regressionTest.sh old mode 100755 new mode 100644 diff --git a/saveSources.py b/saveSources.py old mode 100755 new mode 100644 diff --git a/tagNounPhrases.py b/tagNounPhrases.py old mode 100755 new mode 100644 diff --git a/tagSpeakers.py b/tagSpeakers.py old mode 100755 new mode 100644 From 446f7ce8bff42b733c494b98a42943ab559b5666 Mon Sep 17 00:00:00 2001 From: Ivy Zheng Date: Mon, 5 Jul 2021 19:55:09 +1000 Subject: [PATCH 2/4] Fixed bugs in Tagging and Annotation Normalization Tagging: remove StartZ.is_(None) condition. Annotation: add missing parentheses in StartText and LengthText selection. --- NVivo.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/NVivo.py b/NVivo.py index 1db1d6c..a70db58 100644 --- a/NVivo.py +++ b/NVivo.py @@ -881,7 +881,7 @@ def build_tagging_or_annotation(item): #nvivoNodeReference.c.ReferenceTypeId == literal_column('0'), nvivoItem.c.Id == nvivoNodeReference.c.Node_Item_Id, nvivoItem.c.TypeId == literal_column(NVivo.ItemType.Node), - nvivoNodeReference.c.StartZ.is_(None) + #nvivoNodeReference.c.StartZ.is_(None) )))] for tagging in taggings: build_tagging_or_annotation(tagging) @@ -903,13 +903,13 @@ def build_tagging_or_annotation(item): nvivoAnnotation.c.CreatedDate, nvivoAnnotation.c.ModifiedBy, nvivoAnnotation.c.ModifiedDate - ] + [ + ] + ([ nvivoAnnotation.c.StartText, nvivoAnnotation.c.LengthText ] if args.mac else [ nvivoAnnotation.c.StartX, nvivoAnnotation.c.LengthX - ]))] + ])))] for annotation in annotations: annotation['Node'] = None From c21011271eab5b7210db7a5d490a71fe3e00bea1 Mon Sep 17 00:00:00 2001 From: Ivy Zheng Date: Mon, 5 Jul 2021 20:00:02 +1000 Subject: [PATCH 3/4] Add --drop flag for NormaliseNVP Add --drop flag for dropping/keeping NVivo DB --- NormaliseNVP.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/NormaliseNVP.py b/NormaliseNVP.py index 5aeeaf3..9c660ff 100644 --- a/NormaliseNVP.py +++ b/NormaliseNVP.py @@ -40,6 +40,8 @@ def NormaliseNVP(arglist): help="Port of Microsoft SQL Server") parser.add_argument('-i', '--instance', type=str, help="Microsoft SQL Server instance") + parser.add_argument('-d', '--drop', choices=["keep", "drop"], default="drop", + help='Drop attached NVivo SQL Server Database') parser.add_argument('-u', '--users', choices=["skip", "merge", "overwrite", "replace"], default="merge", help='User action.') @@ -127,7 +129,7 @@ def executecommand(command): verbosity = args.verbosity) # Get reasonably distinct yet recognisable DB name - dbname = 'nvivo' + str(os.getpid()) + dbname = 'nvivo' + str(os.getpid()) + '_' + args.infile.rsplit('\\', 1)[-1].rsplit('/', 1)[-1].rsplit('.', 1)[0].strip() mssqlapi.attach(args.infile, dbname) try: @@ -142,7 +144,11 @@ def executecommand(command): raise finally: - mssqlapi.drop(dbname) + if args.drop == 'drop': + mssqlapi.drop(dbname) + print(f"Drop database {dbname}") + else: + print(f"Keep database {dbname}") if __name__ == '__main__': NormaliseNVP(None) From 15591aa2b86709161e0a9467880d7bdb7a90f7a6 Mon Sep 17 00:00:00 2001 From: Ivy Zheng Date: Fri, 9 Jul 2021 00:25:12 +1000 Subject: [PATCH 4/4] Update NVivo.py Convert StartY from 0-based to 1-based in Tagging.Fragment. --- NVivo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NVivo.py b/NVivo.py index a70db58..eacf0e0 100644 --- a/NVivo.py +++ b/NVivo.py @@ -847,7 +847,7 @@ def build_tagging_or_annotation(item): if item['StartX'] is not None and item['LengthX'] is not None: item['Fragment'] += str(item['StartX']+1) + ':' + str(item['StartX'] + item['LengthX']); if item['StartY'] is not None: - item['Fragment'] += ',' + str(item['StartY']+1) + item['Fragment'] += ',' + str(item['StartY']) if item['LengthY'] > 0: item['Fragment'] += ':' + str(item['StartY'] + item['LengthY'])