From 339ea4b30f6bf25fed248b7814e45a3b509df1f2 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Wed, 6 Sep 2017 21:10:10 -0500 Subject: [PATCH 01/10] adding graphviz diagram and postgres YAML example --- doc/Makefile | 4 ++++ doc/diagram.gv | 19 +++++++++++++++++++ doc/diagram.pdf | Bin 0 -> 15698 bytes tests/configs/postgres.yaml | 28 ++++++++++++++++++++++++++++ 4 files changed, 51 insertions(+) create mode 100644 doc/Makefile create mode 100644 doc/diagram.gv create mode 100644 doc/diagram.pdf create mode 100644 tests/configs/postgres.yaml diff --git a/doc/Makefile b/doc/Makefile new file mode 100644 index 0000000..906fb22 --- /dev/null +++ b/doc/Makefile @@ -0,0 +1,4 @@ +all: diagram.pdf + +diagram.pdf: diagram.gv + dot diagram.gv -Tpdf -odiagram.pdf diff --git a/doc/diagram.gv b/doc/diagram.gv new file mode 100644 index 0000000..1ada956 --- /dev/null +++ b/doc/diagram.gv @@ -0,0 +1,19 @@ +digraph cornet { + pg [label="PostgreSQL"]; + my [label="MySQL"]; + co [label="Cornet"]; + sq [label="Sqoop"]; + h [label="Hadoop"]; + ms [label="SQL Server"; color=gray;]; + + // to be added + ms->co [style=dotted; color=forestgreen;]; + + // schema flows + pg->co [color=forestgreen;]; + my->co [color=forestgreen;]; + co -> sq [label="Commands"; fontcolor=slategray; color=slategray;]; + ms -> sq; + my -> sq; + pg -> sq -> h; +} diff --git a/doc/diagram.pdf b/doc/diagram.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ed3a329aac04670a3d3680d58658f74c8c9719ac GIT binary patch literal 15698 zcmbum1yEhf(twM*yR(7d?i$=(L(s5sch^90cMa}t!7aE1cXtTxt`Bm~x&OWY{j1)q zdTX)QUenVvJw4I`)O_TMqT=+-4D4{^b;lKtaI63(fQ`O693LNmQP#-X#L*PM_9jw> z0{{Sw;%1hPM)q&tmOw`%Q6mEzLnAnTemDn5dn2F~oa^^;bvYXXPK35Y^*t_+DpKV& zb!@S)EYO2RL1b~HP9A3^*SJM$&8xNBn8{)J)`e@Za%`qRJf+S4EwbN(+6$EEHHa9N zFunQ4jExwNPZKZZ0=P979^O?p{H~8@I|HL0TjfvQ_n>L7foE8u9cfFqv%EH^2yTy~ z%5)XNnfMDUx+U9uk)%i!clu2bOMVv2`&+ZSdqv`yr1CO@bUZq<;0?7XtwAg!=A8}q zTyDz@w}LETsI`-eJ>_jgx}A@|@I@xvPPxT(mSh%Ma8v++8oe8N%7d#%!)cH3KE4q* zyQ|Cc0ygssnVhPHjQF8bnfMn|--5^RH}|&!CmrdvDmcCuW^$a`tfrpY+$imY=b}QC zz^u^Sl_;%?n)7+saN8miW;Dz*WN;=FsU@MMh*{C?r7X4eqv8Pg9G za_b>U=XK)D9p&z-0TZar4VSTSXqrWWW}Kjk?8N?p+a^%Q1`RF|Uqb+IT|yVB9x!DP z^*I2Ux;Zh;_*(IF(sr28ZYU~@tueAGbVc~v(NrjLF%(F=Y7Rn`Gz0dhWB705#%9?q z*)-mxLC%Q;4Y*#cbSZVS&Kc=LB^VcYGVM#+ObV}(P#wL2h4A4x%Tws#+fp#pP20Jj zTt|ZO*|9%tNs4}w?V|F9KtbVpxkdYpgbRz(kb{{rL6I99%$6=umu-k*f*?(+xe4uF zidHzQEjx+5kdV_e6Wtr0FieV#iwXR$YxBgK}1+r|THs;_%pEaz2jCkwTbc4r` zpuR|VS-3p@LXzn@9&=>PgMhu|AauAHWdCE3P|E5C} zfQ6hJVmfcn8ILj3PP`l-g*37u1@EZ@hs7hvA79Wm z`2)jIN_a?ka9HShGZjj%W14sW@bIIZDm*6g_tsf-m!h^*=(~ z^i>D_IvAeBCc<1VC2aH5rGm|q)`vaW0Jnz4-6r)fAT*2-XN1YZz9@?zRn)CR$u_i; zO$!SbImBYYf?64yoJ0sLL7DJ0wISK!nfijLWn5R6Z=w*&p22EXi4Q$x$cZgk%Wx21 zetGl7z!T%VY}ejm!Ps+TT?-1K{9E!>%A!`Ct!4B$&{+NgN}lB~j}E%pHZF^>cKmh1 zByo(UuOTShHoUT0(XeP^n^z8Mm_TljCV$nB^r0VQHhZ;XLlUdhQ!RUA02Ku7p{NTS zHvyF2Wd10-J4J zP3mbJRWz0-m`&UQLm-eVF{Rl6+B|7@=m|by`CJ^%$lCDlX!`c{9*5t9@B8;#_+@5g z<9t_mPqO?gO#cbYVy=!7DvobK8NkQ)E_k~bflO}+0HcTyfEmDO0DODCh3dDAzn+ZZ zHr9^+rfp2mM9&0Z|0mSHDZTytZH(hz#sG|}Znj1MMn#~B(I3N#KzpONdYJzy$0%oH zXa*FvaRq3-iI@Ov%uE1QHg15HoFUas8`V|JP3UZsxBokr}|u!Nv5)rN0C$0A?m`&VQG%0@&Hu|6_AJZZT5T zKAdF_mf67j-nFtq-w5GS2lB4X>3H2&RT&11CbdG%D`wu7k6 zB7D)KqG@({Ow)tkBm6;0)o{pMRv!SZQWgKLjTRlBl~zD;aA-KPPa9$&;&d<+5 z<=S3GK-RwLl+hjePQQ|zVL3rB@o+8r9_v7UMY92FY0 zhFk*4FS9HsAS7J&{ekBAT6|Hj$WhS1$N++=h-ZoH>}|0>)+HcLv9oRo`cjMiiO3lV zP@Z2}N+tjwc8_oL0^NJ0Kl-L;!Zbd!fOz!#Qejg7 z^=kwoObv?zWU3#1*%7EuX-7ck?rT>lMaAf+48f0|b{n$6G+ytXoQ%$PA)>#vmCvp1 zOH(-O0mA%45^B3MUkr`(pVKg~UCtmLn5ykA!BXbk7QR}R*_5AGJAggEh``*fA%TR| zgv`9sof5~^-0UI=w);4pbF{ziLkw)bFuH)7z1ngTo(WuQ#1m+j5C633yZ@E(P^MjG zGwR7({j9$90{xog_L_NrjZ7aR5dXq)@T!>|pH_aHS)2Eql>XJmo244MT?4KNL8?&s@b=_@6-hI(9sb0Y@x4I( zGj=u*8H{&HbKzS8p?2AeYZOSz{AZ2V)4b;Vgp`Pn^SwJG3qJOG9^*c%iDHwh!=Q#n zHlT*TH=$Cwc+*@#Kd&JSj`l)c0m0j?K2-hA^Z*^`vOHaX_yp$Y=x`0*eysD87;0S) zx~7pUwe{ml!&xS(bq1%%OMvUDYf1+yXSCRc%~fLY9rmUXWDV78#Qaw}8XqzwK-l7! z;M4NaZN@8!CAuu1pAFJy0_$4Ey#fH@=-v~XftybjugG_w4$qM9>RtPNesWS5`UuJc z!Y%~>b4wpyK-V;#o}stBmOl81+%(|3J;T0knI$)I^LDsBjvZg$6um@@l&{i(LV5U& zfFO68pqTZyh^mtppI?G0T%fSY4XJaXRxu{{r^jEi-GB`D%~Z3e(;N8l>>RBH6H&T1 z#7pShy0(db9%A*cx*Dx`BKtBExh}8QNLP4XuCY}3b^GOtYs7#P<`jLfkDIi67@2C~ z*ATif{29p*1WDAOHZCQmhn4OnQ{Ol~87E9?lyrsVTpOIuJo%&OQ6n{m6^a}K%|3!G zUo4D+@r+jZgEs82dt4&TXuvYWGmd^PNdpHs*`}XjD1WIwl4E_D)v3B+Y(tVguak7A zP6Q^H^#&0Kcy>qZKVZxTWzi7e-HN@)q|1Y9kB1gqlR0achBL{hQ)7P zR8iND@P62eOmb9ah{&R@cCpE^a_ak>kvj;pVVvyfMVd}La_wX~ZFjaO;}%jyEBDZy zS54KZa*wEB;zx8KMgDusrqz3GRrp?A2FP>S$;7W<^|%)qZjPS%1rXYKW{dA7~bK z8ccf`614&`l57fLV@m`-r#OpRFi`U2(XP9OaxpEF#>t0h%($O6=RmUJ70D2ozMm2f zx$E*t_8+IWIXzorS;mJRij%fB2{Ko<%(u`FgJopNk*}RMo-@~Azz+$1HG%IbHav$it zm$$fYZ^khPRZPT~(vG~2UEtMa`|J7(ci&-TZY8-NV`0q7B66Eng}EQ;Ge!Qwj7pQ` z&}yyhFu5d*>0+zuM`zV*{#j$!0gl$>eIxa6hF^nrlR>eC3pGN5+E`=05M_b8t_7nj zbjRs2Eqp!ES1zLp{oKIQ;?At}v+{F7do5bFW;?Bmw{e1OuG|t<-@;?){1sv9g;ajg zm6gRvO~v>4LqvLrweHl`mAau86;x}x;a=!N0()KW>jF6bbq`@>EO>7Jy80hsj8np@ zQyvZhEg9*`TCR)ys+%(o8n9JbsFnsArq8Z?k79<1;W+KO&s9- z7Qb~(olSa;tXQcO>JoRf=Q^xO36)iD`G+H>MPa{X0S@~uti!JZ9V4tfdqU0uOXFBijJos{Lj?svo%83zS^?C7abK~ddaHw`Pl3?YO$z4B4}HF z78|t1K`nB2&(d|mvo!uPG6H#Nm^((`Eed5+vk0JIig)_K0CvT7GCgZ<8&;}gg6-AN zj9%3B7zh~HFbE@a&zK&Dio<(sVJAX#ruQKk@*-u>bb?(78grZ_OLye;7?{06gkJ{z zeV_4F#S~5EDrLJCu`JaZyIW@Qc&HxeK6n7bS3LlELVcW}wM!HujXwrspKF-kqki90 z^(3@#S^*;^5HnbK>a9+IM&@mW$&mY~5A5j(N{Mc+zkgX^Bf6lK5H23P3ID_!Pe5-Y z84@qKm}rs$&eu4Wpy{%fSz!k}R#;0c^8WMHz3D`eDyWJst{JfoA#h9DJs z!7)*r@oJ%BNRqAJe;#iJH`fd5%I>4wHf+l;e?G<-1O^@ZOF8#n32EHDd1j@I1F}YB z2z*3B?QUglJsMk69&@4KmbM7>r>2`166slRo%fRw&oemaeU(hcVqog}4=xAb?jAaZfD#i!F6-`Kg*-Ke3|70Xuz+v1s z2N}8ZmE9lD40q*pGh-*&E`@(B$EVC@m6k?4BoHWV0b9w%-yOVCcV+LFlU7mi*G2v0i$xjgFBOs7#*^9*tcP!JUV5tzY1_^F-uw0%gwy<8@1XiNhwI(jwB;WXY| zhOeleuHg`uT32y9cU@UDn1yq3B`Mr*n9-P1$M&;EVEJi; zA5S&Ikc8^jiWM*2Tn1b-ik%s-pzzn|flboQ>*Bpy2UR}n^|v)rR7Q?mG7#U4FmlLl z(~(9yd}b{9#IQg-8;kJYbV+!UB%Xv4{%-Titl}0+W`yXJ~6 zUz~Psp%xNGT#G&wq^p3O@&%uvh^3jyB;qF;7{c2_c0r&t7k<<7_c2#* zn=CeKK6l&=P>!1I{+k6Gl=!nL#!%|?R$!7I4lmZq1v~hH5h>DaO6U_pvCGt}FH6GMAZZ!11 zHl*)WB&VK~7<?-B30(-@$(h^DE;^^#wK11(fG6KHJO0Bf?w0! zi6#|@CD4;?VI#1NlJa|L*A}~dc*GzROX2*6ftZ^J*d+*Upb_ZN5(+2Vr1A1c&WR3; zCAyX34kjfhR^1oL09}1;@Op>#MOQ{SF!zGaQ@n!msb!;q=_WiQgvjG7K@B(dHq>a_ zujPg&>GQ56v^AAr7Oa=g1Sz_*W;NN(n`Pf|U@kR#xzC1;+Un^tk;~a-AsiETb<}Y_ zFg;cHbzTQ;AWJMV8x|s$6n-&1%+mT1`QYOcmUm7Us0y>R-Bf}Xyr^Y%)~YCokWk>C2t!8E(Hvq4s>w$Z+wqa2^a<$qqvzJeu+ zq$PIbho!c2O9JIzy$pn9@rg zdeRlRBz#ykS}@wKEphDg*RI>mX&Mf5pgT=n3feHL%y-w2l53J{?&n+dv_x8AK|?ei ztzQqYEBNZ@t4x+Lj-dA+Hc>9w=rq@V>_UN@pibW8|P~r+`$)Jsfh>snaF~{H*&YF0O5gZA!SD4tM=E_nI$u!6TQJ(Xac{l0_f+uqn;1%5Blfp&ohdCoX-%FaNH7 z$*0i8;WD@>aJu9y0v8bgH$`RYX^y;y4l&saN4m=Oe%_GkvC6vQD0@L6_m3~D_b13T zw!W=D)C-S3QWSD)lIovynK#1h{bq25M{E&V$sITm z(m5Gs`4W?eE~lZ0xz9Vnu#Jf(n>BzC`hC%q>{$K&r(VHQ^c@n5;Aj-DNy>_LN z#C_O4)dDe69=a>}1pj!NCKDSbK%opt#F-TD)X$9{ggmq9SK8jF(gwpn)*2j+# zfPg*_ML=TsTXCNzGe|6a4&J3p$4Ioe<5D4Xrb8WHxvqut@RR&QL|+b)g9Kj&%)QFK z;QBU#ULP-N$uv4!c>s+Fj9n~l?>yzi0x|h5zL;gGCnI#%@*R$hWTZ0%ayKTaKkl)H zAHGh=*LoQ;mNcBu?a7BY!QMz=X!y{kY3FtW+AKqZ=D;bdtaW)SL*@{7U8S{3x)>Cw zz`%%!y1DiW5(=vq2fHzN^++*niYH-u7L^!$oR^D3giM%F@Snz?e+NAB4h~yAvhEI4 z-l#E$NTeZ6RnkeI*c(f79;q_)Urouh2$a!$dPvb8cubjQq*;nbHPQPhUz*TU5)_+884HJv%Wx4l)FRxNV?d;8Nm)yx5IJOa}LS{&^=6q|My z>#{B&PO>$VUu82ex`Bw0Z2--};#3+oVA>x>8`cUpu2MoU)$(y>l)uD1c>^Cyhe_yw zkRY#Vdthb}2g5u*zRShbeZH0bV^gt0Etz`c>Vc-Jfnree;#c>u9_BHDcK)t{1ke|| zm+9nobY(r&pM4sBQb+>dCdCC3th$hE_?JMs9wFpw<#|?{3$g+q+41JAO6q4{kj_er zJ3jKM3e){!LL~c!(VGe>j=-B+NY9ujcdyZ!wxe+LAcn!Hs;a7GC(1Hi1plmW_a*e( zmbrb(ho0Y~!Zi09i)Uc84Z5FXVhF_$thH-eeQJ&;PNr&PdoM<&dw3oqn77Zy@PxC` z*KiTwy!G?j&{B0pTWGOw<Eney*GBYYzE(%*_BUujtXaA9iN3n zhOToD4Aw|*hMn$+t%R)&#*SU)pZ@LC@``XGdWfAr2-5G(31rbJS7UtP9kbhOzaw9qlM_^lj#Usrsi{p7KyR(T|sh_GHsqYpA_?G!a4=>7Kgm}>6ji_soI zaowq5^oxeFam6hCEeRj9MN=LULv`(SB=VkKOrGi@a)$R#`@{6MMg zO{EIrX^t29Hg#UV$xA8&6jHTwHRvDL39UDkOI9j^S@;R1*Ag;;VFMj*~=xy8_o#S z7lyCG}wg%y13TWtqfF0{X(x+8aAG4&Up^mI*&|CN$sNg%0Lst ze2-GgIgV*Jf$UZnW6K(lj^k;3>D)pFF=VwZ$HIrrtcPXdh9NvY%Mf(y(t>^Gf*wqv z)A^$p4g#d4s}EwnU($tcspG_;E%F02$ne%8ap|pYXYR@ZRybjIg?zYYE26&AT(&z! zGZ8gcYo}P>qQWnU-1Tg7@KJsJekJrwcY@Cn>vYVCQ&XIAIzTNrr|r&_vFG^HTXrUR z>CB2OHv4%*hGhdf)1^U{y(~=Wr{P&bOKu7n@Uly1&jCkHmhBAU0FJXl!`&b1(WR;1 zP``0QN|sM>42|GGV>^oxr*r zwRc@o4wlA0My$mRs2%ZI7%XRNS0weT3wUb^G%Bg>{mH4dayBc@_H0zP~jbp z`So=|%&7bD&X1phQ$cquZ`Bj461>uxKp+Q-Tqp}!rx;hU60+^{u85MDx3#(VQgj#; zG5gMJ5&3B?=Ov7QjPr_?!I#RdaqJ$ohR$2CfU{%D~1~l|=?B^vOc$ja3C@ zD=GA~A!zqjY?h8nDQV-Qf5VC}D&s0*-#PNz3ZgXwfBFJJ*7k6Fon@(vEWb<;=x5us zuvAq(aq?A<-Hsn{Fh}c4&}?#;T6=V>HX|h)et5dXZHY$3brPv;KQYO<-BsP*V)ZLm zB4LdZ#wRj<$o>;aav^f{(J+Qn>S-RuuDYlC2UC$bXH5!G#ZWt39wr~UbComp`Q<=g zs^+0tql@UOtfoZiAK&8rDGrHf#vaj3Vouu`Q7ATq_O-o4WEMDFw}Cf~iH^X#Um6Urf;HT@ab#z92`DG(= z6))P9PIMmAmjQ`*?^)|>?faO(V(-7TX0rG32bP~4ORL+Vu$A0|FGXeSFZ2E>HhmXr zNo#ZwAnS3E-FaDPuYyQBpiO3qF;S&BW*6B+CSyGIR2=DRh)oG*f4-5zQah%vOln+% z>y3w6bos|R-tJzki6j3AS2rhK+>3mq&lH_=>h1A%KlyHY;N2umYcPXP`s&qIY{$kF6E{}&GW$7L z$`>8<)2Z?CPRcNF{RXveVktf16DjO7X}bHHRwfG_-DB<~f2PL9L+6i$%#Hy}X}h*g zt=aJy;8Gy3`QbRjnNRS+nwS&O9JczQXUF4dQ<@(u_4Z9ysI+zpc3d*O8jkNCuuL{p zX=b3lQ6&ZB4W}eU#ETkM2}@vy=rC3EwoM;Ck^gcOZ0*bXsqobOD7!W_(lhczQ6?%7 z7c6I!SRdrcwxFiho!qFVh`he#E2~bbv%rv~Y}Ueo3+E+A_jZhE-}B7Z(jRWO$sfP_ zK+Fj$iRSkezQ@+rnIFDY41Rzwk=al_IbcSq{{2kQQ2BP#`al?YW zv?h)6oBDT}Cf7MpZe-FgbdNx{o|TySD+zHM_!t?1kW5R@y6G+Y)V8 z&R`V+5ITWyzJ{d_dkY;lUzwM5(Q24#G(VU&qfDobqsjs;wO?@cTZKOu>#}%Q7404IZ0)!Wd2LHW;XN=%boI5BZ*Znf#=tmtm__)S+j*as7B-vw77sy)r{q^P z*nswERL*+-ha=CZaN_9@=hhLIr%ZvDMm-+f<9q8>oz*Ym^lQoaK4$oJdD`*MM_o|Q z@sixVw;J53<4QCRV)Ii#3cfxdhiU^CE`^w-tqT<&n|s139Gwq~SWGY%Ib=?1M~{SP zGtC$TlM5y-;`z>mtFE`oKuvv@wa9~;uu4qxM1;p(_Mp4(fC^3|N(SRP!FODN7oi;0 zt*JsB;$zj0@xeg{2mS@Gy^Rq`YYw&Vh&%%$vvaH+iZI^OxnWJ}tBdL>SoQ2}m8uKf z;SAsOkP6siQbLE#tdBFZlLN5KSaQ{;zke3`KsX3eWOE>4xt5*C<4`P8#u?&g<4|vo zaH+-0+O`~AOH4bwg%@V9sklv@{A{~UN_L>73vF|c z%1=QoY9H}t8Z#n4Lwem}$hK6UM_5!}i9%FKr9eGR2!SS=*;RK0m2ctt*p@>P^R3f} zL6BhZBN|J2~A!=r0H|IxW2;+Gat7h!?9E^d4`^l!UGMP2vnal1`k z!CIPAez8XuVVO#Z-q}H)6Vdy&QqfD#$VLvMnt;7XfBO+Z#vN)g<4n^y&0pGqO(03h z`W4eAV&G>p?eO-@6AD;uB{*a*hCg#`{eR|h1};;vN;DtzyXqa*d{auIB4^qp3Zam6 z9I&DUy0*oq7|lqac1;cXTC1Xe>gqVQKVUhdquO zdYUFLq^2cTAz<8LtQXs<)c!8%@?3b;h@u%NmQ#BnQhpskjXm*F zrZ2;b_~vQgCw@H~vw$oQqbe+2IRs*_7~KZ9`RO3CYtE}&+gv=(aG2!$e3q61K%*IP ztIUGL8)%UHH3lngykXNj4XPH2KR5c(Wx7!l4t0hzGRM-LN#%M&R)^nE)J)Z#Wp&7( z4c}pcxPCcJV4+yZF^`Mz(w)L%o9}Dm{=)?Dd)e)KLXv9&3bmM`J4h-VXQU(bpV+5rW!h1zniBS$c}9NlQ%L+sd5Hcvx!f+#-KI_Y>XbGX;}MB|KaI zN_BmMdR19Jm6KsA*zq$=!f1Ix8jbd`(aWEtCh=yIMA^NqgFsITQOcj%`x;ZXS}5P} zYE`8mr#McLJyM;R`v_E;?0~wQQB|RHyg$6l6c2}47*LMg#Y=I z1`IX};TCz05G0-gt%zn$Xss6J3YSVOHIaDb=fyt%ERPTfuBM%~TA2||uVlRpPG1(t z)5}k0$Y1V4JNfw&8~L6}_()tpXfA#)W=NtACCfvi_XOx|_H6Lr&l^_TL44|6sQ+5E zi-5yNEqTyy%3z>@MZqgDqpjF!C-=R3vb!mKpupo|lkvOO3VLP`a46YwatL0u>>e9m z$X?s^Go0Iyp~$eZa8klFg6dKoO^Cd#|MTbKO=GOMA!!2lW^Y909DLW;Z_RWr0(tNR z#TI5ik~_6CaJ$aZe(Xbu`C3ySHu-03y99LYY<0HouNY>>V;Nj zbrXj;v3LuFl+AX$%vvuu;~pcHgP!46FjgAvh0Yjasl+(_JTQjMZWz9;wy1AKkLot3 zsg%tWNMzi_iS*REZ z1nc5nvA%0bn)paIu$Q8(%n&+x8aLq#-%H!;7POK$5#PL7T=K}rV%CI&&p6s0p=|wi z3@jHsYTSw}OSo3mbp9sU*7~7swsRsX56Ww&GU4MzUiIn>PymrAb;-vRo`S+JT`a0Q zY!>DMaA%Nl1If@v7{cMwBHe1KQr`z{4zkrXtwRY}V&Wp%ElDgE9d%07hJ-HeLvhwscA^&h-)Y>uuMRKq3B#w8TNlPL2)_ob8idPvl&A@xwJh#5*oM?Vop z;!6mytfJB_BYItsO{r(LJs|**a0hbKY~_FSyqwcr`pXwq4oMWO6C!oMH>GG<8JL@c zH+A5Jg<|YO9QQ7-SA5Ek4K-m;3=0{B_8H*jNMMibDe&W%lIV|-`M530?OW}^N1Fci zSWX;wo@y>%N;-xhdedqrr(;MXn}Hm@JX&Uo+&TxHCFn9(DbAh@@n(2%=Vywhqt&5- zBZN8kWiLetOc$L?K~c92Tcub?#ux13xJB3ifAF*0WjX{8ugv2*vviz=j*6GO{KN;< z34tS(8S&gkjLqe_j@rxf{y z^7G(5sZp<#`k~joB%6#562Fe|QpyXVsXl7BB`cvPLqa-g z;i~3CO7jy2>qE*FJWzIHI~Un?yRfM*FFOrZxInuA^$m{4h^pr~N!NNyK!f{GJ`w#8 zh?2M7<+DUlsRfHG(Xg=cV&#Ft}sGqRPOz<$jD!D6*VkS5$(ej z+N=a294;*s=9rYSSwH4}40K*bUYWR1eH0su6^9!hQG(pn8capUd4&32Kg4X0)TLNZ zoX=-12|-iXuId(nZOyw=;?~LK-iE^Eh}#?K;)eXFY(f4dX)sqG4#Rk|?Xbg|(2_BY zTNqoHzn3~rI#|l;v#dxNSU{=OHm^*%LscHVdt4jiFac%=X*L;E5c8$tVuhCG@lrRqcB~OU@t64LuwS@`BOv?MvIvnL#NtbS*}v_Td^Rg5t*?TA z%|s-j2Bbf?0|O&T+d7e-SdXh&iDGEoA;Cyo7j<8pc4&;+G9Lu#1C35_qelSfp%_x^ z!zR#AO})XR+t*1&CAYPts+?aIql!dLAoO2 zUp#x#T-}~F6qNAoV&W$+;v=%u8uDYtc9t%yvZgc0>a+AoXYl;O!qV+{lta--Ll!9= zCg{N!?H%AV^L`)7FWS&4YN_$dOJj#Y5X%coaC#lD5O@9Re#=i|5G3kO(9IzQ)D_jk z^M%^{M8DDH3BwMXVag(dF}#K4Lsi?5*)R37$vdzFy5z;4w^({%SdnewRV?i-LnHNu z6&KSi@yJBdGO6*#tV&Q6Wq-?j-r3s=mp8+WRVx4sFvOGDG2|Mc6XIsu5tF=Phh=LW%yo-aZar)zXX`Dc3&SLc7%Ih*)-psL|6}`W4}`ADD~iWC5Q#_EMkN~%&MdEWOn7u zfg&;CQ>fWLn-2-%KifIVzYHE>XI~q;Q1wQibh)gigcEe+$wY4Ch2T_>;WTYoj%w`- zcW!HNAVS&f=e3)BJXUx}`3-YDMm~_F3-dq?M)+WTv9!s!jg@H1{wxOs30TGrHS^|d z)OM!cnS&#u75EVYP0MS@D7UJIt|=L-HNeKM@t6riKIw3=OZ%hWO${yw98xHgNBq(c zSOBt|nA$vFHCsMZvswYnQNY=0wyoEH(5Q5!51A|Vp+ zi;*$f8&3NUR?;pbf#fG22z5NrS_EQ+@hJGZVrkCOG426BZ)p2Jm#12QI%z_tsLED< zWma2&i70HBJ-Vl|F9kEj%+jsg(;xYbW6V=FpUV%AZ*Aog#O4&F2Db7LsYeGF~ zeO{_cCs_ZjtZms}87*XH5$RwXiGoebqM4{CMkF{(ho5F^u6CufTu-z|p5F!5BD8dH z2swYeg8`e1ibgT3?YDmR_0yC$(59TnQEZWGPUNbuVM|7Vte?o)TwdI2IBHI^=ExLb z?a|d<$KfUrYnz|s_B1bWucAYz4rI+(M4D~bAS0Vu(S}KR@v@`%$4urSzIFM4Ci>jb zETI5X_Z8WwYC(%6yEU^l3N?;8v@xmG2e{OVyAl8rv+>3tR4)alyX#4$Fy9wf)6|GU zAK_s5w5$F=wA7iFj+#bq=Ku!&jS>YJ^K>wqGg`8kZ|-8EKd!CR6 z@NHZgb#hvV>qx}l5tNOYa1ur-3;N~xsPQ0=5Uqo+!$iZmM&)t&!7gFJBQX*!-EOITAxG{0E_H5(eN?h9586YoyLCXJ1!`mX#8f z5T%tjv(k5RP`0s>x8athS2i+n`b*CeXyO21`&Sv^zd#cDH`)R~&%*VFUA#>c05=QU z-#m$wBhb>!K*-v}(g^Sud?Dmu@Xn*Wsr;$ry&doA_riY!B0yV7BQq1z|4~tKG_v{x z_?zZ7WVg*^#A|n;CSznziv*Jcd+LD=6Gum7uWmxeFKTuIRJm3Z_mTVW+4@d0IsEA*88gH8K6-Dh^1pZ`r+2vN-=HDpcW3_@dyzNn^=*m+s0@H+ z_BH@!24*$}E&!FOqoXYk;~R4ICN!}J+M1dfI561Qn^05#v7=~jW9Ve?hSC0iFZLGU z{zkEw|DDhMYrTm8-&TQ*$zOH9d-FGq^*@DN?Tw7#m;lUda7_QY037V>tn2_|z+WJx%;0oCMM?pLyz^XqyIzitti`DQ~yKuR+95=0RESZ^S|se zvvR-Bg#XZclfA`&|0QE%<^11mVP<3H{x3cDH}C#;S@u7(=N}u6_Q1D=ZvTgRRxxuo ydhe@u+)LTU=574{hgOraHhvrD_c`!qRyjBV?H&L0E;Bpp+ZIMnE~X$3_x}LG*4|eD literal 0 HcmV?d00001 diff --git a/tests/configs/postgres.yaml b/tests/configs/postgres.yaml new file mode 100644 index 0000000..d9359ed --- /dev/null +++ b/tests/configs/postgres.yaml @@ -0,0 +1,28 @@ +global: + source: + host: localhost + hive: + db: fms + map_types: + java: + UUID: String + hive: + VARBINARY: String + LONGBLOB: String + UUID: String + sqoop_args: + m: 2 + direct: true + data-warehouse: /user/sqoop + skip_tables: + - schema_version + +tasks: + - source: + db: fms + port: 5432 + user: fms + password_file: /home/fms/.password + driver: postgresql + hive: + table_prefix: b_ From 8c0959d452e8b06e4d86de2e91a5af5bdf3c5a3d Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Wed, 6 Sep 2017 21:28:07 -0500 Subject: [PATCH 02/10] adding postgres example schema --- doc/posgres.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/posgres.sql diff --git a/doc/posgres.sql b/doc/posgres.sql new file mode 100644 index 0000000..d155cad --- /dev/null +++ b/doc/posgres.sql @@ -0,0 +1,11 @@ +BEGIN; +CREATE TABLE people ( + id SERIAL PRIMARY KEY NOT NULL, + name TEXT +); +CREATE TABLE addresses ( + id SERIAL PRIMARY KEY NOT NULL, + person INTEGER NOT NULL REFERENCES people(id), + address TEXT NOT NULL +); +COMMIT; From e65e083c562e5bd0b8d5df294154661b006866e2 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Wed, 6 Sep 2017 23:49:25 -0500 Subject: [PATCH 03/10] cleaning up doc folder, which is really more samples, and adding first pass at mssql. looks like will have to upgrade whole tree to be py3 --- cornet/connectors/__init__.py | 3 +++ cornet/connectors/mssql_connector.py | 33 ++++++++++++++++++++++++++++ doc/mssql.yaml | 27 +++++++++++++++++++++++ doc/{posgres.sql => postgres.sql} | 1 + {tests/configs => doc}/postgres.yaml | 0 5 files changed, 64 insertions(+) create mode 100644 cornet/connectors/mssql_connector.py create mode 100644 doc/mssql.yaml rename doc/{posgres.sql => postgres.sql} (86%) rename {tests/configs => doc}/postgres.yaml (100%) diff --git a/cornet/connectors/__init__.py b/cornet/connectors/__init__.py index a758e25..04ca422 100644 --- a/cornet/connectors/__init__.py +++ b/cornet/connectors/__init__.py @@ -12,5 +12,8 @@ def get_connector(source): elif driver == 'postgresql': from postgresql_connector import PostgreSqlConnector return PostgreSqlConnector(source) + elif driver == 'mssql': + from mssql_connector import MSSqlConnector + return MSSqlConnector(source) else: raise LookupError("Driver {0} not supported".format(driver)) diff --git a/cornet/connectors/mssql_connector.py b/cornet/connectors/mssql_connector.py new file mode 100644 index 0000000..a39a9aa --- /dev/null +++ b/cornet/connectors/mssql_connector.py @@ -0,0 +1,33 @@ +import pymssql # yes I know, would be nice to use JDBC connector instead +from base_connector import BaseConnector +from cornet.connectors import Table, Column + + +class MSSqlConnector(BaseConnector): + + jdbc_url_prefix = 'jdbc:sqlserver' + + def _get_db_conn(self): + source = self.source + return pymssql.connect( + host=source['host'], + user=source['user'], + password=self._get_password(), + database=source['db']) + + def get_tables(self): + sql = """ + select table_name, table_type + from information_schema.tables + where table_name NOT IN ('sysdiagrams'); """ + res = self.query(sql) + return map(Table._make, res) + + def get_columns(self, table): + sql = """ + select column_name, upper(udt_name) + from information_schema.columns + where table_catalog = '{0}' + and table_name = '{1}'; """ + res = self.query(sql.format(self.source['db'], table.name)) + return map(Column._make, res) diff --git a/doc/mssql.yaml b/doc/mssql.yaml new file mode 100644 index 0000000..b51cf63 --- /dev/null +++ b/doc/mssql.yaml @@ -0,0 +1,27 @@ +global: + source: + host: 192.168.122.107 + hive: + db: test + map_types: + java: + UUID: String + hive: + VARBINARY: String + LONGBLOB: String + UUID: String + sqoop_args: + m: 2 + direct: true + data-warehouse: /user/sqoop + skip_tables: + - schema_version + +tasks: + - source: + db: test + user: WORKGROUP\Administrator + password_file: /home/fms/.password + driver: mssql + hive: + table_prefix: mssql_ diff --git a/doc/posgres.sql b/doc/postgres.sql similarity index 86% rename from doc/posgres.sql rename to doc/postgres.sql index d155cad..a413593 100644 --- a/doc/posgres.sql +++ b/doc/postgres.sql @@ -8,4 +8,5 @@ CREATE TABLE addresses ( person INTEGER NOT NULL REFERENCES people(id), address TEXT NOT NULL ); +-- CREATE VIEW people_addresses AS COMMIT; diff --git a/tests/configs/postgres.yaml b/doc/postgres.yaml similarity index 100% rename from tests/configs/postgres.yaml rename to doc/postgres.yaml From 17ecf0ab5302f110e053259896881f284ff6a8b5 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Wed, 6 Sep 2017 23:53:02 -0500 Subject: [PATCH 04/10] 2to3, now semi-working MSSQL --- cornet/connectors/__init__.py | 6 +++--- cornet/connectors/mssql_connector.py | 6 +++--- cornet/connectors/mysql_connector.py | 6 +++--- cornet/connectors/postgresql_connector.py | 6 +++--- cornet/main.py | 14 +++++++------- cornet/sqoop_cmd.py | 4 ++-- cornet/task_config.py | 2 +- cornet/utils.py | 6 +++--- 8 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cornet/connectors/__init__.py b/cornet/connectors/__init__.py index 04ca422..e5a9291 100644 --- a/cornet/connectors/__init__.py +++ b/cornet/connectors/__init__.py @@ -7,13 +7,13 @@ def get_connector(source): driver = source['driver'] if driver == 'mysql': - from mysql_connector import MySqlConnector + from .mysql_connector import MySqlConnector return MySqlConnector(source) elif driver == 'postgresql': - from postgresql_connector import PostgreSqlConnector + from .postgresql_connector import PostgreSqlConnector return PostgreSqlConnector(source) elif driver == 'mssql': - from mssql_connector import MSSqlConnector + from .mssql_connector import MSSqlConnector return MSSqlConnector(source) else: raise LookupError("Driver {0} not supported".format(driver)) diff --git a/cornet/connectors/mssql_connector.py b/cornet/connectors/mssql_connector.py index a39a9aa..ad7ff35 100644 --- a/cornet/connectors/mssql_connector.py +++ b/cornet/connectors/mssql_connector.py @@ -1,5 +1,5 @@ import pymssql # yes I know, would be nice to use JDBC connector instead -from base_connector import BaseConnector +from .base_connector import BaseConnector from cornet.connectors import Table, Column @@ -21,7 +21,7 @@ def get_tables(self): from information_schema.tables where table_name NOT IN ('sysdiagrams'); """ res = self.query(sql) - return map(Table._make, res) + return list(map(Table._make, res)) def get_columns(self, table): sql = """ @@ -30,4 +30,4 @@ def get_columns(self, table): where table_catalog = '{0}' and table_name = '{1}'; """ res = self.query(sql.format(self.source['db'], table.name)) - return map(Column._make, res) + return list(map(Column._make, res)) diff --git a/cornet/connectors/mysql_connector.py b/cornet/connectors/mysql_connector.py index 842dff9..0b9696c 100644 --- a/cornet/connectors/mysql_connector.py +++ b/cornet/connectors/mysql_connector.py @@ -1,5 +1,5 @@ import MySQLdb -from base_connector import BaseConnector +from .base_connector import BaseConnector from cornet.connectors import Table, Column @@ -18,7 +18,7 @@ def _get_db_conn(self): def get_tables(self): res = self.query("show full tables") - return map(Table._make, res) + return list(map(Table._make, res)) def get_columns(self, table): sql = """ @@ -27,4 +27,4 @@ def get_columns(self, table): where table_schema = '{0}' and table_name = '{1}' """ res = self.query(sql.format(self.source['db'], table.name)) - return map(Column._make, res) + return list(map(Column._make, res)) diff --git a/cornet/connectors/postgresql_connector.py b/cornet/connectors/postgresql_connector.py index d95b872..df09f2d 100644 --- a/cornet/connectors/postgresql_connector.py +++ b/cornet/connectors/postgresql_connector.py @@ -1,5 +1,5 @@ import psycopg2 -from base_connector import BaseConnector +from .base_connector import BaseConnector from cornet.connectors import Table, Column @@ -22,7 +22,7 @@ def get_tables(self): from information_schema.tables where table_schema NOT IN ('pg_catalog', 'information_schema'); """ res = self.query(sql) - return map(Table._make, res) + return list(map(Table._make, res)) def get_columns(self, table): sql = """ @@ -31,4 +31,4 @@ def get_columns(self, table): where table_catalog = '{0}' and table_name = '{1}'; """ res = self.query(sql.format(self.source['db'], table.name)) - return map(Column._make, res) + return list(map(Column._make, res)) diff --git a/cornet/main.py b/cornet/main.py index 0477bb7..20d8e81 100644 --- a/cornet/main.py +++ b/cornet/main.py @@ -1,8 +1,8 @@ import click -from connectors import get_connector -from task_config import TaskConfig -from sqoop_cmd import SqoopCmd -from utils import match_any +from .connectors import get_connector +from .task_config import TaskConfig +from .sqoop_cmd import SqoopCmd +from .utils import match_any def print_sqoop_cmds(task): @@ -11,7 +11,7 @@ def print_sqoop_cmds(task): for table in sorted(to_import, key=lambda tbl: tbl.name): columns = conn.get_columns(table) cmd = SqoopCmd(task, table, columns) - print cmd.as_string() + print(cmd.as_string()) def get_tables_to_import(conn, task): @@ -26,10 +26,10 @@ def print_schema(task): with get_connector(task.source) as conn: to_import = get_tables_to_import(conn, task) for table in to_import: - print '\n=== {0}.{1} ==='.format(task.source['db'], table[0]) + print('\n=== {0}.{1} ==='.format(task.source['db'], table[0])) columns = conn.get_columns(table) for c in columns: - print '{0}: {1}'.format(c.name, c.type) + print('{0}: {1}'.format(c.name, c.type)) @click.command() diff --git a/cornet/sqoop_cmd.py b/cornet/sqoop_cmd.py index c2800cf..3835317 100644 --- a/cornet/sqoop_cmd.py +++ b/cornet/sqoop_cmd.py @@ -1,4 +1,4 @@ -from utils import merge_dict +from .utils import merge_dict class SqoopCmd: @@ -56,7 +56,7 @@ def _arg2str(k, v): def as_string(self): args = { SqoopCmd._arg2str(k, v) - for k, v in self.args().iteritems() + for k, v in self.args().items() if v} name = 'sqoop import' return ' \\\n '.join([name] + sorted(args)) + '\n' diff --git a/cornet/task_config.py b/cornet/task_config.py index b900a55..c27ea1b 100644 --- a/cornet/task_config.py +++ b/cornet/task_config.py @@ -1,6 +1,6 @@ import yaml from jinja2 import Environment, FileSystemLoader -from utils import merge_dict, dict_without_key +from .utils import merge_dict, dict_without_key import os.path diff --git a/cornet/utils.py b/cornet/utils.py index 2aec2fe..a3d9702 100644 --- a/cornet/utils.py +++ b/cornet/utils.py @@ -12,10 +12,10 @@ def merge_dict(a, b): assert isinstance(b, dict), "Second arg not a dict, but {0} ".format(b) merged = {} - for key in set(a.keys() + b.keys()): - if key not in b.keys(): + for key in set(list(a.keys()) + list(b.keys())): + if key not in list(b.keys()): merged[key] = a[key] - elif key not in a.keys(): + elif key not in list(a.keys()): merged[key] = b[key] elif isinstance(a[key], dict) and isinstance(b[key], dict): merged[key] = merge_dict(a[key], b[key]) From ff13ff10cbff76b98c63d2ee6e692186c6ebe1fd Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 00:01:55 -0500 Subject: [PATCH 05/10] I do not have ability to actually test sqoop, but seems to pull from MSSQL. Only works on Py3 for unknown reasons relating to pymssql. Probably should emit warnings on py2. --- cornet/connectors/mssql_connector.py | 2 +- doc/mssql.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cornet/connectors/mssql_connector.py b/cornet/connectors/mssql_connector.py index ad7ff35..1682ac1 100644 --- a/cornet/connectors/mssql_connector.py +++ b/cornet/connectors/mssql_connector.py @@ -25,7 +25,7 @@ def get_tables(self): def get_columns(self, table): sql = """ - select column_name, upper(udt_name) + select column_name, upper(data_type) from information_schema.columns where table_catalog = '{0}' and table_name = '{1}'; """ diff --git a/doc/mssql.yaml b/doc/mssql.yaml index b51cf63..06f76df 100644 --- a/doc/mssql.yaml +++ b/doc/mssql.yaml @@ -22,6 +22,7 @@ tasks: db: test user: WORKGROUP\Administrator password_file: /home/fms/.password + port: 1143 driver: mssql hive: table_prefix: mssql_ From add9fad8ba24d498fabf97fd699c17f8b4fd14e3 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 00:14:31 -0500 Subject: [PATCH 06/10] this is a dirty hack of a warning, but should do the business for now --- cornet/connectors/mssql_connector.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cornet/connectors/mssql_connector.py b/cornet/connectors/mssql_connector.py index 1682ac1..afc7308 100644 --- a/cornet/connectors/mssql_connector.py +++ b/cornet/connectors/mssql_connector.py @@ -1,4 +1,5 @@ import pymssql # yes I know, would be nice to use JDBC connector instead +import sys from .base_connector import BaseConnector from cornet.connectors import Table, Column @@ -7,6 +8,9 @@ class MSSqlConnector(BaseConnector): jdbc_url_prefix = 'jdbc:sqlserver' + if sys.version_info[0] < 3: + raise("Sorry, MSSQL isn't yet happy on Python 2.") + def _get_db_conn(self): source = self.source return pymssql.connect( From e8ade9cef9249c9620fa5089815e0122e3307bae Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 00:22:30 -0500 Subject: [PATCH 07/10] fixing inline comment for Travis --- cornet/connectors/mssql_connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cornet/connectors/mssql_connector.py b/cornet/connectors/mssql_connector.py index afc7308..f756a0a 100644 --- a/cornet/connectors/mssql_connector.py +++ b/cornet/connectors/mssql_connector.py @@ -1,4 +1,4 @@ -import pymssql # yes I know, would be nice to use JDBC connector instead +import pymssql # yes I know, would be nice to use JDBC connector instead import sys from .base_connector import BaseConnector from cornet.connectors import Table, Column From 30c7d040494e70243198b1d6bc811cd1b6a2dcb3 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 13:01:07 -0500 Subject: [PATCH 08/10] more travis testing --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d4c07ce..3daf3e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,3 +9,5 @@ after_success: codecov env: - TOXENV=py27 + - TOXENV=py35 + - TOXENV=py36 From 1c72518b0bbb9c8d7d94b5e5cbe7cf231262d108 Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 13:05:15 -0500 Subject: [PATCH 09/10] updating tox for travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3daf3e3..d37023d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python install: - - pip install tox==1.8.1 + - pip install tox==2.8.1 - pip install coveralls codecov script: - tox From 489e1e62530350c26b183f382730b86d01337e7a Mon Sep 17 00:00:00 2001 From: "Franklin M. Siler" Date: Thu, 7 Sep 2017 13:10:38 -0500 Subject: [PATCH 10/10] more travis/tox updates --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d37023d..20abaae 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python install: - - pip install tox==2.8.1 + - pip install tox - pip install coveralls codecov script: - tox @@ -10,4 +10,4 @@ after_success: env: - TOXENV=py27 - TOXENV=py35 - - TOXENV=py36 + - TOXENV=py36-dev