(org.apache.sysds.runtime.matrix.operators.ReorgOperator, org.apache.sysds.runtime.matrix.data.MatrixValue, int, int, int)" because "inBlock" is null #358
Description
Script:
X = read($1)
y = read($2)
XtX = t(X) %*% X; # 500 x 500
Xty = t(X) %*% y; # 500 x 1
R = solve(XtX, Xty)
print(sum(R!=0))
z = t(y) %*% X;
print(sum(z!=0))
write(R, $3, format="binary")Program plan
# EXPLAIN (RUNTIME):
# Memory Budget local/remote = 2100MB/?MB/?MB
# Degree of Parallelism (vcores) local/remote = 8/
PROGRAM ( size CP/MR = 33/0 )
--MAIN PROGRAM
----GENERIC (lines 22-31) [recompile=false]
------CP createvar pREADX target/testTemp/functions/ooc/lmDSTest/in/X false MATRIX binary 10000 500 1000 5000000 copy
------CP createvar pREADy target/testTemp/functions/ooc/lmDSTest/in/y false MATRIX binary 10000 1 1000 10000 copy
------CP createvar _mVar0 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp0 true MATRIX binary 10000 500 1000 5000000 copy
------OOC rblk pREADX.MATRIX.FP64 _mVar0.MATRIX.FP64 1000 true
------CP createvar _mVar1 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp1 true MATRIX binary -1 -1 -1 -1 copy
------OOC tee _mVar0.MATRIX.FP64 _mVar1.MATRIX.FP64
------CP rmvar _mVar0
------CP createvar _mVar2 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp2 true MATRIX binary 500 500 1000 -1 copy
------OOC tsmm _mVar1.MATRIX.FP64 _mVar2.MATRIX.FP64 LEFT
------CP createvar _mVar3 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp3 true MATRIX binary 10000 1 1000 10000 copy
------OOC rblk pREADy.MATRIX.FP64 _mVar3.MATRIX.FP64 1000 true
------CP createvar _mVar4 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp4 true MATRIX binary -1 -1 -1 -1 copy
------OOC tee _mVar3.MATRIX.FP64 _mVar4.MATRIX.FP64
------CP rmvar _mVar3
------CP createvar _mVar5 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp5 true MATRIX binary 1 10000 1000 10000 copy
------OOC r' _mVar4.MATRIX.FP64 _mVar5.MATRIX.FP64
------CP createvar _mVar6 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp6 true MATRIX binary 1 500 1000 -1 copy
------OOC ba+* _mVar5.MATRIX.FP64 _mVar1.MATRIX.FP64 _mVar6.MATRIX.FP64 8
------CP rmvar _mVar5
------CP createvar _mVar7 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp7 true MATRIX binary 500 1 1000 -1 copy
------CP r' _mVar6.MATRIX.FP64 _mVar7.MATRIX.FP64 8
------CP rmvar _mVar6
------CP createvar _mVar8 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp8 true MATRIX binary 500 1 1000 -1 copy
------CP solve _mVar2.MATRIX.FP64 _mVar7.MATRIX.FP64 _mVar8.MATRIX.FP64 8
------CP rmvar _mVar2 _mVar7
------CP createvar _mVar9 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp9 true MATRIX binary 500 1 1000 -1 copy
------OOC != _mVar8.MATRIX.FP64 0.SCALAR.INT64.true _mVar9.MATRIX.FP64
------OOC uak+ _mVar9.MATRIX.FP64 _Var10.SCALAR.STRING
------CP rmvar _mVar9
------CP createvar _mVar11 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp10 true MATRIX binary 1 10000 1000 10000 copy
------OOC r' _mVar4.MATRIX.FP64 _mVar11.MATRIX.FP64
------CP rmvar _mVar4
------CP createvar _mVar12 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp11 true MATRIX binary 1 500 1000 -1 copy
------OOC ba+* _mVar11.MATRIX.FP64 _mVar1.MATRIX.FP64 _mVar12.MATRIX.FP64 8
------CP rmvar _mVar11 _mVar1
------CP createvar _mVar13 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p25068_10.120.110.127//_t0/temp12 true MATRIX binary 1 500 1000 -1 copy
------OOC != _mVar12.MATRIX.FP64 0.SCALAR.INT64.true _mVar13.MATRIX.FP64
------CP rmvar _mVar12
------OOC uak+ _mVar13.MATRIX.FP64 _Var14.SCALAR.STRING
------CP rmvar _mVar13
------CP print _Var10.SCALAR.STRING.false _Var15.SCALAR.STRING 8
------CP rmvar _Var10
------CP print _Var14.SCALAR.STRING.false _Var16.SCALAR.STRING 8
------CP rmvar _Var14
------CP write _mVar8.MATRIX.FP64 target/testTemp/functions/ooc/lmDSTest/out/R.SCALAR.STRING.true binary.SCALAR.STRING.true .SCALAR.STRING.true 1000
------CP rmvar _mVar8 _Var15 _Var16ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=0
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=1
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=2
BUFFER: 12000027/9437184 size=3
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=3
BUFFER: 12000027/9437184 size=4
BUFFER: 12000027/9437184 size=5
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=4
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=5
BUFFER: 12000027/9437184 size=6
BUFFER: 12000027/9437184 size=6
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=6
BUFFER: 12000027/9437184 size=7
BUFFER: 12000027/9437184 size=7
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=7
BUFFER: 12000027/9437184 size=8
BUFFER: 12000027/9437184 size=8
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=8
BUFFER: 12000027/9437184 size=9
BUFFER: 12000027/9437184 size=9
BUFFER: 12000027/9437184 size=9
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=9
BUFFER: 12000027/9437184 size=10
BUFFER: 12000027/9437184 size=10
BUFFER: 12000027/9437184 size=10
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=10
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=0
ResettableStream: cacheInProgress=false, replayPosition=0, numBlocks=10
ResettableStream: cacheInProgress=true, replayPosition=0, numBlocks=1
BUFFER: 12016045/9437184 size=12
BUFFER: 12016045/9437184 size=12
BUFFER: 12016045/9437184 size=12
ResettableStream: cacheInProgress=false, replayPosition=1, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=2, numBlocks=10
BUFFER: 12016045/9437184 size=12
ResettableStream: cacheInProgress=false, replayPosition=3, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=4, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=5, numBlocks=10
BUFFER: 20016063/9437184 size=12
ResettableStream: cacheInProgress=false, replayPosition=6, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=7, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=8, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=9, numBlocks=10
ResettableStream: cacheInProgress=false, replayPosition=10, numBlocks=10
BUFFER: 32016090/9437184 size=12
BUFFER: 28016081/9437184 size=12
BUFFER: 24016072/9437184 size=12
BUFFER: 20016063/9437184 size=12
BUFFER: 16016054/9437184 size=12
BUFFER: 12016045/9437184 size=12
BUFFER: 12008036/9437184 size=12
SystemDS Statistics:
Total elapsed time: 2.865 sec.
Total compilation time: 1.583 sec.
Total execution time: 1.282 sec.
Cache hits (Mem/Li/WB/FS/HDFS): 0/0/0/0/0.
Cache writes (Li/WB/FS/HDFS): 0/1/0/0.
Cache times (ACQr/m, RLS, EXP): 0.143/0.004/0.004/0.000 sec.
HOP DAGs recompiled (PRED, SB): 0/0.
HOP DAGs recompile time: 0.000 sec.
Total JIT compile time: 1.549 sec.
Total JVM GC count: 1.
Total JVM GC time: 0.032 sec.
Heavy hitter instructions:
# Instruction Time(s) Count
1 ooc_tsmm 0.940 1
2 ooc_ba+* 0.172 1
3 createvar 0.043 10
4 ooc_rblk 0.002 2
5 ooc_r' 0.001 1
6 rmvar 0.000 3
7 ooc_tee 0.000 2
org.apache.sysds.runtime.DMLRuntimeException: org.apache.sysds.runtime.DMLRuntimeException: ERROR: Runtime error in program block generated from statement block between lines 22 and 31 -- Error evaluating instruction: CP°r'°_mVar6·MATRIX·FP64°_mVar7·MATRIX·FP64°8
at org.apache.sysds.runtime.controlprogram.Program.execute(Program.java:163)
at org.apache.sysds.api.ScriptExecutorUtils.executeRuntimeProgram(ScriptExecutorUtils.java:89)
at org.apache.sysds.api.DMLScript.execute(DMLScript.java:497)
at org.apache.sysds.api.DMLScript.executeScript(DMLScript.java:333)
at org.apache.sysds.test.AutomatedTestBase.main(AutomatedTestBase.java:1547)
at org.apache.sysds.test.AutomatedTestBase.runTestWithTimeout(AutomatedTestBase.java:1502)
at org.apache.sysds.test.AutomatedTestBase.lambda$runTest$0(AutomatedTestBase.java:1412)
at java.base/java.lang.Thread.run(Thread.java:840)
Caused by: org.apache.sysds.runtime.DMLRuntimeException: ERROR: Runtime error in program block generated from statement block between lines 22 and 31 -- Error evaluating instruction: CP°r'°_mVar6·MATRIX·FP64°_mVar7·MATRIX·FP64°8
at org.apache.sysds.runtime.controlprogram.ProgramBlock.executeSingleInstruction(ProgramBlock.java:269)
at org.apache.sysds.runtime.controlprogram.ProgramBlock.executeInstructions(ProgramBlock.java:196)
at org.apache.sysds.runtime.controlprogram.BasicProgramBlock.execute(BasicProgramBlock.java:127)
at org.apache.sysds.runtime.controlprogram.Program.execute(Program.java:157)
... 7 more
Caused by: org.apache.sysds.runtime.DMLRuntimeException: org.apache.sysds.runtime.DMLRuntimeException: org.apache.sysds.runtime.DMLRuntimeException: java.lang.NullPointerException: Cannot invoke "org.apache.sysds.runtime.matrix.data.MatrixBlock.reorgOperations(org.apache.sysds.runtime.matrix.operators.ReorgOperator, org.apache.sysds.runtime.matrix.data.MatrixValue, int, int, int)" because "inBlock" is null
at org.apache.sysds.runtime.controlprogram.caching.MatrixObject.readBlobFromStream(MatrixObject.java:549)
at org.apache.sysds.runtime.controlprogram.caching.MatrixObject.readBlobFromStream(MatrixObject.java:73)
at org.apache.sysds.runtime.controlprogram.caching.CacheableData.acquireReadIntern(CacheableData.java:636)
at org.apache.sysds.runtime.controlprogram.caching.CacheableData.acquireRead(CacheableData.java:575)
at org.apache.sysds.runtime.controlprogram.context.ExecutionContext.getMatrixInput(ExecutionContext.java:346)
at org.apache.sysds.runtime.instructions.cp.ReorgCPInstruction.processInstruction(ReorgCPInstruction.java:150)
at org.apache.sysds.runtime.controlprogram.ProgramBlock.executeSingleInstruction(ProgramBlock.java:233)
... 10 more
Caused by: org.apache.sysds.runtime.DMLRuntimeException: org.apache.sysds.runtime.DMLRuntimeException: java.lang.NullPointerException: Cannot invoke "org.apache.sysds.runtime.matrix.data.MatrixBlock.reorgOperations(org.apache.sysds.runtime.matrix.operators.ReorgOperator, org.apache.sysds.runtime.matrix.data.MatrixValue, int, int, int)" because "inBlock" is null
at org.apache.sysds.runtime.instructions.ooc.MatrixVectorBinaryOOCInstruction.lambda$processInstruction$0(MatrixVectorBinaryOOCInstruction.java:127)
at org.apache.sysds.runtime.instructions.ooc.OOCInstruction.lambda$oocTask$0(OOCInstruction.java:109)
at java.base/java.util.concurrent.ForkJoinTask$AdaptedRunnableAction.exec(ForkJoinTask.java:1375)
at java.base/java.util.concurrent.ForkJoinTask.doExec$$$capture(ForkJoinTask.java:373)
at java.base/java.util.concurrent.ForkJoinTask.doExec(ForkJoinTask.java)
at java.base/java.util.concurrent.ForkJoinPool$WorkQueue.topLevelExec(ForkJoinPool.java:1182)
at java.base/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1655)
at java.base/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1622)
at java.base/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165)
Caused by: org.apache.sysds.runtime.DMLRuntimeException: java.lang.NullPointerException: Cannot invoke "org.apache.sysds.runtime.matrix.data.MatrixBlock.reorgOperations(org.apache.sysds.runtime.matrix.operators.ReorgOperator, org.apache.sysds.runtime.matrix.data.MatrixValue, int, int, int)" because "inBlock" is null
at org.apache.sysds.runtime.instructions.ooc.TransposeOOCInstruction.lambda$processInstruction$0(TransposeOOCInstruction.java:74)
... 8 more
Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.sysds.runtime.matrix.data.MatrixBlock.reorgOperations(org.apache.sysds.runtime.matrix.operators.ReorgOperator, org.apache.sysds.runtime.matrix.data.MatrixValue, int, int, int)" because "inBlock" is null
at org.apache.sysds.runtime.instructions.ooc.TransposeOOCInstruction.lambda$processInstruction$0(TransposeOOCInstruction.java:68)
... 8 more