test for lmds

[j143]

Algorithm

X = read($1)
y = read($2)

XtX = t(X) %*% X; # 500 x 10000 -- 10000 x 500 == 500 x 500
Xty = t(X) %*% y; # 500 x 10000 -- 10000 x 1 == 500 x 1
R = solve(XtX, Xty)
write(R, $3, format="binary")

Program plan

# EXPLAIN (RUNTIME):
# Memory Budget local/remote = 2100MB/?MB/?MB
# Degree of Parallelism (vcores) local/remote = 8/
PROGRAM ( size CP/MR = 20/0 )
--MAIN PROGRAM
----GENERIC (lines 22-28) [recompile=false]
------CP createvar pREADX target/testTemp/functions/ooc/lmDSTest/in/X false MATRIX binary 100000 500 1000 35001023 copy
------CP createvar pREADy target/testTemp/functions/ooc/lmDSTest/in/y false MATRIX binary 100000 1 1000 100000 copy
------CP createvar _mVar45 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp36 true MATRIX binary 100000 500 1000 35001023 copy
------OOC rblk pREADX.MATRIX.FP64 _mVar45.MATRIX.FP64 1000 true
------CP createvar _mVar46 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp37 true MATRIX binary -1 -1 -1 -1 copy
------OOC tee _mVar45.MATRIX.FP64 _mVar46.MATRIX.FP64
------CP rmvar _mVar45
------CP createvar _mVar47 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp38 true MATRIX binary 500 500 1000 -1 copy
------OOC tsmm _mVar46.MATRIX.FP64 _mVar47.MATRIX.FP64 LEFT
------CP createvar _mVar48 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp39 true MATRIX binary 100000 1 1000 100000 copy
------OOC rblk pREADy.MATRIX.FP64 _mVar48.MATRIX.FP64 1000 true
------CP createvar _mVar49 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp40 true MATRIX binary 1 100000 1000 100000 copy
------CP r' _mVar48.MATRIX.FP64 _mVar49.MATRIX.FP64 8
------CP rmvar _mVar48
------CP createvar _mVar50 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp41 true MATRIX binary 1 500 1000 -1 copy
------OOC ba+* _mVar49.MATRIX.FP64 _mVar46.MATRIX.FP64 _mVar50.MATRIX.FP64 8
------CP rmvar _mVar49 _mVar46
------CP createvar _mVar51 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp42 true MATRIX binary 500 1 1000 -1 copy
------CP r' _mVar50.MATRIX.FP64 _mVar51.MATRIX.FP64 8
------CP rmvar _mVar50
------CP createvar _mVar52 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp43 true MATRIX binary 500 1 1000 -1 copy
------CP solve _mVar47.MATRIX.FP64 _mVar51.MATRIX.FP64 _mVar52.MATRIX.FP64 8
------CP rmvar _mVar47 _mVar51
------CP write _mVar52.MATRIX.FP64 target/testTemp/functions/ooc/lmDSTest/out/R.SCALAR.STRING.true binary.SCALAR.STRING.true .SCALAR.STRING.true 1000
------CP rmvar _mVar52

SystemDS Statistics:
Total elapsed time:		7.102 sec.
Total compilation time:		0.169 sec.
Total execution time:		6.934 sec.
Cache hits (Mem/Li/WB/FS/HDFS):	2/0/0/0/0.
Cache writes (Li/WB/FS/HDFS):	0/2/0/1.
Cache times (ACQr/m, RLS, EXP):	0.401/0.000/0.000/0.036 sec.
HOP DAGs recompiled (PRED, SB):	0/0.
HOP DAGs recompile time:	0.000 sec.
Total JIT compile time:		11.624 sec.
Total JVM GC count:		11.
Total JVM GC time:		0.215 sec.
Heavy hitter instructions:
 #  Instruction  Time(s)  Count
 1  ooc_tsmm       5.656      1
 2  ooc_ba+*       0.771      1
 3  solve          0.409      1
 4  r'             0.061      2
 5  write          0.036      1
 6  rmvar          0.001      6
 7  createvar      0.000     10
 8  ooc_rblk       0.000      2
 9  ooc_tee        0.000      1

(i): 0 ->> expected15.69408196611309, result: 0.0
(i): 1 ->> expected15.26330863435574, result: 0.0
(i): 2 ->> expected17.94563715431772, result: 0.0
(i): 3 ->> expected16.64012771955282, result: 0.0
(i): 4 ->> expected16.14694912406201, result: 0.0
(i): 5 ->> expected15.9135948464932, result: 0.0
(i): 6 ->> expected16.105116111349012, result: 0.0
(i): 7 ->> expected17.692011997638105, result: 0.0
(i): 8 ->> expected17.473401098492218, result: 0.0
(i): 9 ->> expected13.957976193695574, result: 0.0
(i): 10 ->> expected15.242706382304968, result: 0.0
(i): 11 ->> expected17.567865055956215, result: 0.0

[j143]

------OOC rblk pREADy.MATRIX.FP64 _mVar48.MATRIX.FP64 1000 true
------CP createvar _mVar49 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p32028_10.230.136.127//_t0/temp40 true MATRIX binary 1 100000 1000 100000 copy
------CP r' _mVar48.MATRIX.FP64 _mVar49.MATRIX.FP64 8

Heavy hitter instructions:
 #  Instruction  Time(s)  Count
 1  ooc_tsmm       5.656      1
 2  ooc_ba+*       0.771      1
 3  solve          0.409      1
 4  r'             0.061      2
 5  write          0.036      1
 6  rmvar          0.001      6
 7  createvar      0.000     10
 8  ooc_rblk       0.000      2
 9  ooc_tee        0.000      1

(i): 0 ->> expected15.69408196611309, result: 0.0
(i): 1 ->> expected15.26330863435574, result: 0.0
(i): 2 ->> expected17.94563715431772, result: 0.0

with

//Handle Y or actualY for transpose
		Lop yLop = isYTransposed ? actualY.constructLops() : Y.constructLops();
		ExecType inputReorgExecType = (Y.hasFederatedOutput()) ? ExecType.FED :
						((Y.getExecType() == ExecType.OOC) ? ExecType.OOC : ExecType.CP);

------OOC rblk pREADy.MATRIX.FP64 _mVar3.MATRIX.FP64 1000 true
------CP createvar _mVar4 target\testTemp\functions\ooc\lmDSTest\lmDS/target/scratch_space//_p19916_10.230.136.127//_t0/temp4 true MATRIX binary 1 100000 1000 100000 copy
------OOC r' _mVar3.MATRIX.FP64 _mVar4.MATRIX.FP64

Heavy hitter instructions:
  #  Instruction  Time(s)  Count
  1  ooc_tsmm       5.898      1
  2  ooc_ba+*       0.809      1
  3  solve          0.196      1
  4  createvar      0.064     10
  5  r'             0.055      1
  6  write          0.043      1
  7  ooc_rblk       0.002      2
  8  rmvar          0.002      6
  9  ooc_r'         0.001      1
 10  ooc_tee        0.000      1

(i): 0 ->> expected15.69408196611309, result: 0.004011813661929021
(i): 1 ->> expected15.26330863435574, result: 0.003682665913873535
(i): 2 ->> expected17.94563715431772, result: 0.00885535113482066