tee rewrite rule position in programrewrite

[j143]

Overall expectation

(START)
   |
   v
[Read X (from Disk)] --> [Read y (from Disk)]
   |                          |
   v                          v
[OOC rblk (Start Streaming X)] [CP rblk (Load y to Memory)]
   |
   +-----------------------+
   |                       |
   v                       v
[OOC tee (Split Stream)]   |
   |                       |
   +--> [Stream 1]          +--> [Stream 2]
   |                       |
   v                       v
[OOC tsmm]              [OOC r' (Transpose)]
   |                       |
   v                       v
[XtX (Small, in Memory)]  [t(X) Stream] ----+
   |                                        |
   |                                        v
   +----------------------------------> [CP ba+* (Multiply)]
                                          |
                                          v
                                        [Xty (Small, in Memory)]
                                          |
                                          |
   +------------------------------------->+
   |
   v
[CP solve]
   |
   v
[Write R (to Disk)]
   |
   v
(END)

option 1: Rewrite rule order

if( staticRewrites )
		{
			//add static HOP DAG rewrite rules
			_dagRuleSet.add(     new RewriteRemoveReadAfterWrite()               ); //dependency: before blocksize
			_dagRuleSet.add(     new RewriteBlockSizeAndReblock()                );
			_dagRuleSet.add(     new RewriteInjectOOCTee()                       );
			if( OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION )
				_dagRuleSet.add( new RewriteRemoveUnnecessaryCasts()             );

plan

PROGRAM ( size CP/MR = 26/0 )
--MAIN PROGRAM
----GENERIC (lines 22-28) [recompile=false]
------CP createvar pREADX target/testTemp/functions/ooc/lmDSTest/in/X false MATRIX binary 2000 2000 1000 2800370 copy
------CP createvar pREADy target/testTemp/functions/ooc/lmDSTest/in/v false MATRIX binary 2000 1 1000 2000 copy
------CP createvar _mVar0 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp0 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC rblk pREADX.MATRIX.FP64 _mVar0.MATRIX.FP64 1000 true
------CP createvar _mVar1 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp1 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC tee _mVar0.MATRIX.FP64 tee_out_X_0.MATRIX.FP64 tee_out_X_1.MATRIX.FP64
------CP rmvar _mVar0
------CP createvar _mVar2 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp2 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC r' tee_out_X_0.MATRIX.FP64 _mVar2.MATRIX.FP64
------CP rmvar tee_out_X_0
------CP createvar _mVar3 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp3 true MATRIX binary 2000 2000 1000 -1 copy
------OOC ba+* _mVar2.MATRIX.FP64 tee_out_X_1.MATRIX.FP64 _mVar3.MATRIX.FP64 8
------CP rmvar _mVar2 tee_out_X_1
------CP createvar _mVar4 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp4 true MATRIX binary 2000 1 1000 2000 copy
------OOC rblk pREADy.MATRIX.FP64 _mVar4.MATRIX.FP64 1000 true
------CP createvar _mVar5 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp5 true MATRIX binary 1 2000 1000 2000 copy
------CP r' _mVar4.MATRIX.FP64 _mVar5.MATRIX.FP64 8
------CP rmvar _mVar4
------CP createvar _mVar6 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp6 true MATRIX binary 1 2000 1000 -1 copy
------OOC ba+* _mVar5.MATRIX.FP64 tee_out_X_2.MATRIX.FP64 _mVar6.MATRIX.FP64 8
------CP rmvar _mVar5 tee_out_X_2
------CP createvar _mVar7 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp7 true MATRIX binary 2000 1 1000 -1 copy
------CP r' _mVar6.MATRIX.FP64 _mVar7.MATRIX.FP64 8
------CP rmvar _mVar6
------CP createvar _mVar8 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p30140_10.89.140.127//_t0/temp8 true MATRIX binary 2000 1 1000 -1 copy
------CP solve _mVar3.MATRIX.FP64 _mVar7.MATRIX.FP64 _mVar8.MATRIX.FP64 8
------CP rmvar _mVar3 _mVar7
------CP write _mVar8.MATRIX.FP64 target/testTemp/functions/ooc/lmDSTest/out/res.SCALAR.STRING.true binary.SCALAR.STRING.true .SCALAR.STRING.true 1000
------CP rmvar _mVar8
------CP cpvar _mVar1 tee_out_X_0
------CP cpvar _mVar1 tee_out_X_1
------CP mvvar _mVar1 tee_out_X_2

option 2: Rewrite rule order

               if( OptimizerUtils.ALLOW_AUTO_VECTORIZATION )
				_dagRuleSet.add( new RewriteIndexingVectorization()              ); //dependency: cse, simplifications
			_dagRuleSet.add(     new RewriteInjectOOCTee()                       );
			_dagRuleSet.add( new RewriteInjectSparkPReadCheckpointing()          ); //dependency: reblock

plan

PROGRAM ( size CP/MR = 25/0 )
--MAIN PROGRAM
----GENERIC (lines 22-28) [recompile=false]
------CP createvar pREADX target/testTemp/functions/ooc/lmDSTest/in/X false MATRIX binary 2000 2000 1000 2800370 copy
------CP createvar pREADy target/testTemp/functions/ooc/lmDSTest/in/v false MATRIX binary 2000 1 1000 2000 copy
------CP createvar _mVar0 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp0 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC rblk pREADX.MATRIX.FP64 _mVar0.MATRIX.FP64 1000 true
------CP createvar _mVar1 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp1 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC tee _mVar0.MATRIX.FP64 tee_out_X_0.MATRIX.FP64 tee_out_X_1.MATRIX.FP64
------CP rmvar _mVar0
------CP createvar _mVar2 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp2 true MATRIX binary 2000 2000 1000 2800370 copy
------OOC r' tee_out_X_0.MATRIX.FP64 _mVar2.MATRIX.FP64
------CP createvar _mVar3 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp3 true MATRIX binary 2000 2000 1000 -1 copy
------OOC ba+* _mVar2.MATRIX.FP64 tee_out_X_1.MATRIX.FP64 _mVar3.MATRIX.FP64 8
------CP rmvar _mVar2 tee_out_X_1
------CP createvar _mVar4 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp4 true MATRIX binary 2000 1 1000 2000 copy
------OOC rblk pREADy.MATRIX.FP64 _mVar4.MATRIX.FP64 1000 true
------CP createvar _mVar5 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp5 true MATRIX binary 1 2000 1000 2000 copy
------CP r' _mVar4.MATRIX.FP64 _mVar5.MATRIX.FP64 8
------CP rmvar _mVar4
------CP createvar _mVar6 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp6 true MATRIX binary 1 2000 1000 -1 copy
------OOC ba+* _mVar5.MATRIX.FP64 tee_out_X_0.MATRIX.FP64 _mVar6.MATRIX.FP64 8
------CP rmvar _mVar5 tee_out_X_0
------CP createvar _mVar7 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp7 true MATRIX binary 2000 1 1000 -1 copy
------CP r' _mVar6.MATRIX.FP64 _mVar7.MATRIX.FP64 8
------CP rmvar _mVar6
------CP createvar _mVar8 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p37620_10.89.140.127//_t0/temp8 true MATRIX binary 2000 1 1000 -1 copy
------CP solve _mVar3.MATRIX.FP64 _mVar7.MATRIX.FP64 _mVar8.MATRIX.FP64 8
------CP rmvar _mVar3 _mVar7
------CP write _mVar8.MATRIX.FP64 target/testTemp/functions/ooc/lmDSTest/out/res.SCALAR.STRING.true binary.SCALAR.STRING.true .SCALAR.STRING.true 1000
------CP rmvar _mVar8
------CP cpvar _mVar1 tee_out_X_0
------CP cpvar _mVar1 tee_out_X_1
------CP rmvar _mVar1

[j143]

rewrite condition:

// Check if this hop is a candidate for OOC Tee injection
		if (DMLScript.USE_OOC 
			&& hop.getDataType().isMatrix()
			&& !HopRewriteUtils.isData(hop, OpOpData.TEE)
			&& hop.getParent().size() > 1
			&& isSelfTranposePattern(hop)
		) {
			rewriteCandidates.add(hop);
		}

present:

# EXPLAIN (RUNTIME):
# Memory Budget local/remote = 2100MB/?MB/?MB
# Degree of Parallelism (vcores) local/remote = 8/
PROGRAM ( size CP/MR = 20/0 )
--MAIN PROGRAM
----GENERIC (lines 22-28) [recompile=false]
------CP createvar pREADX target/testTemp/functions/ooc/lmDSTest/in/X false MATRIX binary 100000 500 1000 35001023 copy
------CP createvar pREADy target/testTemp/functions/ooc/lmDSTest/in/y false MATRIX binary 100000 1 1000 100000 copy
------CP createvar _mVar0 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp0 true MATRIX binary 100000 500 1000 35001023 copy
------OOC rblk pREADX.MATRIX.FP64 _mVar0.MATRIX.FP64 1000 true
------CP createvar _mVar1 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp1 true MATRIX binary -1 -1 -1 -1 copy
------OOC tee _mVar0.MATRIX.FP64 _mVar1.MATRIX.FP64
------CP rmvar _mVar0
------CP createvar _mVar2 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp2 true MATRIX binary 500 500 1000 -1 copy
------OOC tsmm _mVar1.MATRIX.FP64 _mVar2.MATRIX.FP64 LEFT
------CP createvar _mVar3 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp3 true MATRIX binary 100000 1 1000 100000 copy
------OOC rblk pREADy.MATRIX.FP64 _mVar3.MATRIX.FP64 1000 true
------CP createvar _mVar4 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp4 true MATRIX binary 1 100000 1000 100000 copy
------CP r' _mVar3.MATRIX.FP64 _mVar4.MATRIX.FP64 8
------CP rmvar _mVar3
------CP createvar _mVar5 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp5 true MATRIX binary 1 500 1000 -1 copy
------OOC ba+* _mVar4.MATRIX.FP64 _mVar1.MATRIX.FP64 _mVar5.MATRIX.FP64 8
------CP rmvar _mVar4 _mVar1
------CP createvar _mVar6 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp6 true MATRIX binary 500 1 1000 -1 copy
------CP r' _mVar5.MATRIX.FP64 _mVar6.MATRIX.FP64 8
------CP rmvar _mVar5
------CP createvar _mVar7 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p26420_192.168.1.9//_t0/temp7 true MATRIX binary 500 1 1000 -1 copy
------CP solve _mVar2.MATRIX.FP64 _mVar6.MATRIX.FP64 _mVar7.MATRIX.FP64 8
------CP rmvar _mVar2 _mVar6
------CP write _mVar7.MATRIX.FP64 target/testTemp/functions/ooc/lmDSTest/out/res.SCALAR.STRING.true binary.SCALAR.STRING.true .SCALAR.STRING.true 1000
------CP rmvar _mVar7

SystemDS Statistics:
Total elapsed time:		10.499 sec.
Total compilation time:		6.388 sec.
Total execution time:		4.111 sec.
Cache hits (Mem/Li/WB/FS/HDFS):	2/0/0/0/0.
Cache writes (Li/WB/FS/HDFS):	0/2/0/1.
Cache times (ACQr/m, RLS, EXP):	0.223/0.002/0.002/0.055 sec.
HOP DAGs recompiled (PRED, SB):	0/0.
HOP DAGs recompile time:	0.000 sec.
Total JIT compile time:		1.186 sec.
Total JVM GC count:		10.
Total JVM GC time:		0.201 sec.
Heavy hitter instructions:
 #  Instruction  Time(s)  Count
 1  ooc_tsmm       3.230      1
 2  ooc_ba+*       0.538      1
 3  solve          0.165      1
 4  write          0.062      1
 5  createvar      0.055     10
 6  r'             0.050      2
 7  ooc_rblk       0.002      2
 8  rmvar          0.001      6
 9  ooc_tee        0.000      1