From 3fb74e6ee183b51024d5cd389bbc8b0dc8880fdf Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Mon, 18 Sep 2023 12:02:04 -0400 Subject: [PATCH 1/8] Updated MNRL to the latest with support for MACOS with M Processors --- libs/MNRL | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/MNRL b/libs/MNRL index 85659d0..93a0bfd 160000 --- a/libs/MNRL +++ b/libs/MNRL @@ -1 +1 @@ -Subproject commit 85659d0c669dae863f591fd5b0c383f88af58d2e +Subproject commit 93a0bfde43102da799f73c07e7ba5650c2ccd558 From 303b2d2c1332c41e3a1c28117067affc2b74f651 Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Mon, 18 Sep 2023 12:48:27 -0400 Subject: [PATCH 2/8] Updated installation instructions and example c++ code --- README.md | 46 ++++++---------------------------------------- 1 file changed, 6 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 3cbac55..14d351c 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,20 @@ # Virtual Automata Simulator (VASim) -VASim is a homogeneous non-deterministic finite automata simulator. Homogeneous automata do not contain epsilon transitions, and also have the property that all incoming transitions for a particular state have the same rule. +VASim is a homogeneous non-deterministic finite automata (NFA) simulator. Homogeneous automata do not contain epsilon transitions, and also have the property that all incoming transitions for a particular state have the same symbol set. VASim can parse, transform, simulate, and profile homogeneous automata, and is meant to be an open tool for automata processing research. VASim can also be extended to support arbitrary automata processing elements other than traditional states. ## Installation -VASim is designed to run on 64-bit machines GCC 4.9+. Users have successfully compiled on 32-bit platforms but these builds are not officially supported. Below are the commands used to install on all tested platforms. +VASim is designed to run on 64-bit machines, Intel/AMD x86-64 or Apple's M processors. Below are the commands used to install on all tested platforms. -### Ubuntu Linux (14.04/16.04 with gcc 5.0+) +### Ubuntu Linux (18.04/20.04/22.04) and MacOS (M1 or Intel) ```bash $ git clone https://github.com/jackwadden/VASim.git $ cd VASim $ make -``` - -### MacOS (10.11+) -```bash - -$ brew install nasm -$ brew install gcc5 -$ git clone https://github.com/jackwadden/VASim.git -$ cd VASim -$ make - -``` - -### Windows 7 (Cygwin64) - -Dependencies: -- Cygwin64 -- Git (https://git-scm.com/download/win) - -First you must install all required Cygwin64 software tools -```bash - -C:\> chdir cygwin64 -C:\cygwin64> move . -C:\cygwin64> setup-x86_64.exe -q -P wget -P gcc-g++ -P make -P diffutils -P libmpfr-devel -P libgmp-devel -P libmpc-devel -P nasm - -``` -Once Cygwin64 is configured, VASim should compile correctly in a Cygwin terminal. - -```bash - -$ git clone https://github.com/jackwadden/VASim.git -$ cd VASim -$ make - - ``` ## Command Line Usage @@ -98,7 +62,7 @@ int main(int argc, char * argv[]) { cout << ap.getReportVector().size() << endl; // enable report gathering for the automata - ap.enableReport(); + ap.setReport(true); // initialize simulation ap.initializeSimulation(); @@ -114,6 +78,8 @@ int main(int argc, char * argv[]) { } ``` +To compile against the library, use the Makefile in test/VASim. + ## Issues Please see https://www.github.com/jackwadden/VASim/issues for a list of known bugs or to create an issue ticket. From 95e0940b8d67866a13eee9cf181594c56224b485 Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Tue, 6 Feb 2024 16:07:03 -0500 Subject: [PATCH 3/8] Added objects in src directory to be ignored --- src/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/.gitignore diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..2416a67 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +obj/ From 16af05cf09c2a04dad9415cb4388868e548bcacf Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Tue, 6 Feb 2024 16:09:17 -0500 Subject: [PATCH 4/8] Fixed .gitignot --- src/.gitignore | 2 +- src/.main.cpp.swp | Bin 16384 -> 0 bytes 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 src/.main.cpp.swp diff --git a/src/.gitignore b/src/.gitignore index 2416a67..94053f2 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -1 +1 @@ -obj/ +obj/* diff --git a/src/.main.cpp.swp b/src/.main.cpp.swp deleted file mode 100644 index f8cec71b3bafcbbd6f5ad85ffaeaa656d0468671..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNU5wmT6}~M$Ezp!uf)IhyyHiniv+J2{frK+~ z1lZbdY~Sm9&bjB_b9~P6wx%zvtnnwx(+tN^#(q{-ZuZzBI{a=vHZ z`CrN5C+%zd`fnxu-%i?J-q-(j(%(2D_eONqeg`1d63 z3(5J#ef>#uaVj)WXrRzQp@BjJg$4=@6dEWrP-vjgK%s#`1OJyAFm1*@ftd?Rr1AYf z&HrzHn6X!YUjRP>UI4xc^nni00P4Ug;3V)c@G0P4;2z+|4>EQccm#O$2xDIdP6I!C zfUz$DUj!<^_dmqg_kd@B`+={0kg;1IVC*?y9r)G#j9mn7-N)G5z$?HW@Yeew6YxB+ z1HAb@#!TS1?*$Fu25=Gh3km}bpbmTi_&e$XuK|As{sjCH_yh2J;4<(q@JZk}@NwW6 za1?k0`gt9=2~cg~CEz>2vjFMN1`a(qj+XDcychUw#})i6Z<}r?&agpGAbNhFaynY3 zvbv$;<#b(*9FaG~k-bPYLlro(GaSa4BNeBAvKXfeGw4`T-0GSEKS4J;m$oPP(~O^} z0IYHXD%8@M#vHguuBimZf$$U$l^Lj}BW$9LUvSliYs54ZgMwqhVA%{P5pXs zHB!E3DqoH#S0+DUPVoq`JkeDAL_(YHPk#;#BTR;^8E%8d5w>}A=s^>zJzlyjOAw26 zp0;pqKyJh63b#k?*<+JNan*G~uDeZXbFt^BgH$d{zP3_dn2CEB5y>#vBnCaxx*}AV zrO)dJ0r%Tr76{W0p-67|9+WM()(fV2t3^!fZ z-{ox|qjyB$nmyhUYF7x!6NU$c#grQ)QOYYT(p^bp`BDl?30uqPOCd2Y2yezKY#wAu z`q^4-tTfKoma2;@>*o?p5o<<2Lv7KWVVpE*_u7K)21jGpCq1A!9Pr{!oA>-MbXu+` zA7mVlHH=*hk^@5V#U?3Qm{ymz8Wg?js4h48H4zXSY9Y1uOr=C1L%xgua1vOcbVaZ$ z9zV@n4(2|Nweoz4rQ#Q>wJHxHSA-L{5l2XnwuI3(r5~xYv)E^Id?{|k+~6Fffe!9E zo&$S>Z*U#URAkQ5r|CyhIWDgCE$^DyL0AbF_KXdsx~Als_}$6k$b3;SuXij7#@wYK z*c^>ob-lj4QCq7vRyNk_hu9IEjbQ}B!%lDdLB^D7x{YjsZ`61POBa^;hIISf>_Kzb zs2$gDp}U`}2V<28`#lA1UlWGwcT6|lBDF|rs!1PKz{gI&N~c?$hf%8&n7u9*ppAWk z?Gh-59s^Sod!j=v;4IwKw(~{U7!e^BSIcm?2$=0bF>>8v)U!%jM49Y z+tc%*j?A$UDLA7cKWw;i{G5KBk)fRL_e4;)OqnABOnROs*m7a6`;I))`Z>!9tjJNZ z6}l$O1rFEAZ%A=$!(&00RQHZkkk$tq?T<}kxj8C_p-AuX7e&EZBJYyp0LaaFSx zd%}unMfJK%zPx#^%Ey z#0`=_*i@0%!#fhLE^*mt-K2OWOx|6Y_c3i;t8ZaP#4j4`wGb|{IjbAz%JIvNqMHx_ ziE#J#(A>ftLxaGJOY>Xl%bby9naz*r(l98#3=yPekA)<%N&6_$aVfCsXQPP3#=ndx zPZ(>~f*XZbM4wmP4tAL8dIuEuTWZ{QxPbNf-Lh2qP2G5dlX@Mv??xk!9k*1_3F8$ z8UC4&U!qqWLWgZWo31cc({tx&&l7c8yh@xB&2dng(m4;ZfjCQvLbD^rGuaU3LM(pG zNYfJ`!f8xU*% z_!jUaz=2N#e?{*96M$*~-vyonz5!eZd_V%{fMwt)@G9yAF9Od4+rYh)4*}y7(jaY2v8h_1_})n8YnbSXyAWa z1990m{-rq07OqI^ka6+UX^*E>-3dfV^w98eeLLDjcPJyLVMoJX^wl!rvGGA9(k$^w zrGnB5QY-w~NamA5%rF+EVybCbK5Aw< zMKx_(S7<0=j#C8+$33BtHY<~li*l!|6FB5=;RMq>Vm3Ot;X1Y5ajp$|1{c*Nw?JVV zW%qQtmoS^{OViu2?uZ;DM#)N0(HAvyR22K%b^?@CsmeeBFjXrRDt<1q6d&muDp^t> zZR~^W^njhdpwf%3=YtD6&v2aiER~d-Nr6{4O-`mN1M?+R71J3)XDXgjh;0qwu$1~l zmxy;~W}rF6XJ>h-rc(q`)&gad;S4WPJCYBp+tj&aNcVb>khVLVa2}m>I7yl&ANivP z;tV9YZX9x4*Ak5~AC96a(NPm&S1O{6Qe!9@0WuYBU9m89nSAufK2O)M$z9~q zB3{^rpEHW>ACuk>jWzzzpzSrjEUP_L8ZLf1B0UdBre=;aIRKVEkv+#LTauoVfG~kl zvR@7zRH0F+pgXd{~*Bvj92$noex>HY>6@GMK)1LfR>b0T( z(-Y;k6NKt;4Q4HFax(upNS4syV`qslyGIAbAWVCQF8m$){K1-KcaG-F{tW Date: Tue, 6 Feb 2024 16:13:38 -0500 Subject: [PATCH 5/8] Added root gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dc7ff53 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +src/obj/ From 99e8fcad6bb035e53b310462ca3d32d555ceb6a6 Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Wed, 7 Feb 2024 12:01:14 -0500 Subject: [PATCH 6/8] Added splitting automata --- src/main.cpp | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index e6262d1..deb2654 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -6,6 +6,8 @@ #include #include #include +#include + #include "errno.h" #define FROM_INPUT_STRING false @@ -35,6 +37,7 @@ void usage(char * argv) { printf(" -f, --hdl Output automata as one-hot encoded verilog HDL for execution on an FPGA (EXPERIMENTAL)\n"); printf(" -B, --blif Output automata as .blif circuit for place-and-route using VPR.\n"); printf(" --graph Output automata as .graph file for HyperScan.\n"); + printf(" -S, --split Specify number of seperate automata files to split automata into.\n"); printf("\n OPTIMIZATIONS:\n"); printf(" -O, --optimize-global Run all optimizations on all automata subgraphs.\n"); @@ -101,6 +104,8 @@ int main(int argc, char * argv[]) { uint32_t dump_state_cycle = 0; bool widen = false; bool two_stride = false; + bool split = false; + uint32_t split_count = 0; // long option switches const int32_t graph_switch = 1000; @@ -111,7 +116,7 @@ int main(int argc, char * argv[]) { const int32_t two_stride_switch = 1005; int c; - const char * short_opt = "thsqrbnfcdBDeamxipOLT:P:"; + const char * short_opt = "thsqrbnfcdBDeamxipOLT:P:S:"; struct option long_opt[] = { {"help", no_argument, NULL, 'h'}, @@ -134,6 +139,7 @@ int main(int argc, char * argv[]) { {"remove-ors", no_argument, NULL, 'x'}, {"thread-width", required_argument, NULL, 'T'}, {"thread-height", required_argument, NULL, 'P'}, + {"split", required_argument, NULL, 'S'}, {"graph", no_argument, NULL, graph_switch}, {"enforce-fanin", required_argument, NULL, fanin_switch}, {"enforce-fanout", required_argument, NULL, fanout_switch}, @@ -218,6 +224,11 @@ int main(int argc, char * argv[]) { num_threads_packets = atoi(optarg); break; + case 'S': + split = true; + split_count = atoi(optarg); + break; + case 'D': to_dfa = true; break; @@ -538,6 +549,31 @@ int main(int argc, char * argv[]) { a->automataToHDLFile("automata_" + to_string(counter) + ".v"); } + // Emit as multiple files + if(split) { + + vector components = ap.splitConnectedComponents(); + + int num_components = components.size(); + int num_automata_per_file = num_components / split_count; + int left_overs = num_components % split_count; + + assert(num_components == (num_automata_per_file * split_count + left_overs)); + + for(int i = 0; i < split_count; i++){ + Automata first = Automata(*components[i * num_automata_per_file]); + for(int j = 0; j < num_automata_per_file; j++) + first.unsafeMerge(components[i * num_automata_per_file + j]); + if(i == (split_count - 1)){ + for(int j = split_count * num_automata_per_file; j < num_components; j++){ + first.unsafeMerge(components[j]); + } + } + first.finalizeAutomata(); + first.automataToANMLFile("automata_split_" + std::to_string(i) + ".anml"); + } + } + // Emit as .blif circuit file if(to_blif){ if(!quiet) From 74b24848947f27c67ef39f0422b38c92960cd767 Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Wed, 28 Feb 2024 14:55:03 -0500 Subject: [PATCH 7/8] Updated HLS generator -- removed bit-wise comparisons and did ranging instead --- include/automata.h | 4 +- src/.main.cpp.swp | Bin 16384 -> 0 bytes src/automata.cpp | 451 ++++++++++++++++++++++++++++++++++++++++++++- src/main.cpp | 81 +++++++- 4 files changed, 521 insertions(+), 15 deletions(-) delete mode 100644 src/.main.cpp.swp diff --git a/include/automata.h b/include/automata.h index 3929d15..5b2052f 100644 --- a/include/automata.h +++ b/include/automata.h @@ -1,5 +1,5 @@ /** - * @file + * Header file for VASim */ #ifndef AUTOMATA_H #define AUTOMATA_H @@ -59,7 +59,6 @@ class Automata { std::vector latchedSpecialElements; std::vector activateNoInputSpecialElements; - // Simulation Statistics std::vector> reportVector; std::unordered_map> activationVector; @@ -116,6 +115,7 @@ class Automata { void automataToNFAFile(std::string fn); void automataToANMLFile(std::string fn); void automataToMNRLFile(std::string fn); + void automataToHLSFiles(int N, int split_factor); void automataToHDLFile(std::string fn); void automataToBLIFFile(std::string fn); void automataToGraphFile(std::string fn); diff --git a/src/.main.cpp.swp b/src/.main.cpp.swp deleted file mode 100644 index f8cec71b3bafcbbd6f5ad85ffaeaa656d0468671..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 16384 zcmeHNU5wmT6}~M$Ezp!uf)IhyyHiniv+J2{frK+~ z1lZbdY~Sm9&bjB_b9~P6wx%zvtnnwx(+tN^#(q{-ZuZzBI{a=vHZ z`CrN5C+%zd`fnxu-%i?J-q-(j(%(2D_eONqeg`1d63 z3(5J#ef>#uaVj)WXrRzQp@BjJg$4=@6dEWrP-vjgK%s#`1OJyAFm1*@ftd?Rr1AYf z&HrzHn6X!YUjRP>UI4xc^nni00P4Ug;3V)c@G0P4;2z+|4>EQccm#O$2xDIdP6I!C zfUz$DUj!<^_dmqg_kd@B`+={0kg;1IVC*?y9r)G#j9mn7-N)G5z$?HW@Yeew6YxB+ z1HAb@#!TS1?*$Fu25=Gh3km}bpbmTi_&e$XuK|As{sjCH_yh2J;4<(q@JZk}@NwW6 za1?k0`gt9=2~cg~CEz>2vjFMN1`a(qj+XDcychUw#})i6Z<}r?&agpGAbNhFaynY3 zvbv$;<#b(*9FaG~k-bPYLlro(GaSa4BNeBAvKXfeGw4`T-0GSEKS4J;m$oPP(~O^} z0IYHXD%8@M#vHguuBimZf$$U$l^Lj}BW$9LUvSliYs54ZgMwqhVA%{P5pXs zHB!E3DqoH#S0+DUPVoq`JkeDAL_(YHPk#;#BTR;^8E%8d5w>}A=s^>zJzlyjOAw26 zp0;pqKyJh63b#k?*<+JNan*G~uDeZXbFt^BgH$d{zP3_dn2CEB5y>#vBnCaxx*}AV zrO)dJ0r%Tr76{W0p-67|9+WM()(fV2t3^!fZ z-{ox|qjyB$nmyhUYF7x!6NU$c#grQ)QOYYT(p^bp`BDl?30uqPOCd2Y2yezKY#wAu z`q^4-tTfKoma2;@>*o?p5o<<2Lv7KWVVpE*_u7K)21jGpCq1A!9Pr{!oA>-MbXu+` zA7mVlHH=*hk^@5V#U?3Qm{ymz8Wg?js4h48H4zXSY9Y1uOr=C1L%xgua1vOcbVaZ$ z9zV@n4(2|Nweoz4rQ#Q>wJHxHSA-L{5l2XnwuI3(r5~xYv)E^Id?{|k+~6Fffe!9E zo&$S>Z*U#URAkQ5r|CyhIWDgCE$^DyL0AbF_KXdsx~Als_}$6k$b3;SuXij7#@wYK z*c^>ob-lj4QCq7vRyNk_hu9IEjbQ}B!%lDdLB^D7x{YjsZ`61POBa^;hIISf>_Kzb zs2$gDp}U`}2V<28`#lA1UlWGwcT6|lBDF|rs!1PKz{gI&N~c?$hf%8&n7u9*ppAWk z?Gh-59s^Sod!j=v;4IwKw(~{U7!e^BSIcm?2$=0bF>>8v)U!%jM49Y z+tc%*j?A$UDLA7cKWw;i{G5KBk)fRL_e4;)OqnABOnROs*m7a6`;I))`Z>!9tjJNZ z6}l$O1rFEAZ%A=$!(&00RQHZkkk$tq?T<}kxj8C_p-AuX7e&EZBJYyp0LaaFSx zd%}unMfJK%zPx#^%Ey z#0`=_*i@0%!#fhLE^*mt-K2OWOx|6Y_c3i;t8ZaP#4j4`wGb|{IjbAz%JIvNqMHx_ ziE#J#(A>ftLxaGJOY>Xl%bby9naz*r(l98#3=yPekA)<%N&6_$aVfCsXQPP3#=ndx zPZ(>~f*XZbM4wmP4tAL8dIuEuTWZ{QxPbNf-Lh2qP2G5dlX@Mv??xk!9k*1_3F8$ z8UC4&U!qqWLWgZWo31cc({tx&&l7c8yh@xB&2dng(m4;ZfjCQvLbD^rGuaU3LM(pG zNYfJ`!f8xU*% z_!jUaz=2N#e?{*96M$*~-vyonz5!eZd_V%{fMwt)@G9yAF9Od4+rYh)4*}y7(jaY2v8h_1_})n8YnbSXyAWa z1990m{-rq07OqI^ka6+UX^*E>-3dfV^w98eeLLDjcPJyLVMoJX^wl!rvGGA9(k$^w zrGnB5QY-w~NamA5%rF+EVybCbK5Aw< zMKx_(S7<0=j#C8+$33BtHY<~li*l!|6FB5=;RMq>Vm3Ot;X1Y5ajp$|1{c*Nw?JVV zW%qQtmoS^{OViu2?uZ;DM#)N0(HAvyR22K%b^?@CsmeeBFjXrRDt<1q6d&muDp^t> zZR~^W^njhdpwf%3=YtD6&v2aiER~d-Nr6{4O-`mN1M?+R71J3)XDXgjh;0qwu$1~l zmxy;~W}rF6XJ>h-rc(q`)&gad;S4WPJCYBp+tj&aNcVb>khVLVa2}m>I7yl&ANivP z;tV9YZX9x4*Ak5~AC96a(NPm&S1O{6Qe!9@0WuYBU9m89nSAufK2O)M$z9~q zB3{^rpEHW>ACuk>jWzzzpzSrjEUP_L8ZLf1B0UdBre=;aIRKVEkv+#LTauoVfG~kl zvR@7zRH0F+pgXd{~*Bvj92$noex>HY>6@GMK)1LfR>b0T( z(-Y;k6NKt;4Q4HFax(upNS4syV`qslyGIAbAWVCQF8m$){K1-KcaG-F{tW +//#include using namespace std; using namespace MNRL; @@ -301,10 +303,6 @@ void Automata::rawAddSpecialElement(SpecialElement *specel) { } } - - - - /** * Adds all outputs of ste2 to ste1 then removes ste2 from the automata. Used in common prefix merging algorithm. */ @@ -344,6 +342,7 @@ vector Automata::splitConnectedComponents() { continue; // Create new automata in vector Automata *m = new Automata(); + connectedComponents.push_back(m); index++; @@ -1237,7 +1236,7 @@ void Automata::automataToDotFile(string out_fn) { } - // heatmap color + // heatmap color:0 str.append("\" style=filled fillcolor="); if(profile) { //fillcolor = getElementColorLog(e.first); @@ -1697,6 +1696,444 @@ void Automata::automataToMNRLFile(string out_fn) { } +/** + * Outputs automata to Vitis-HLS + * +**/ + +void writeHeaderFile(int num_components, string return_type, string tree_header) { + // Generate header file + string str = ""; + str += "#ifndef _AUTOMATA_HPP_\n"; + str += "#define _AUTOMATA_HPP_\n"; + str += "\n"; + str += "#include \"../krnl_automata.hpp\"\n"; + str += "\n"; + + // Fill in the tree header if there is one + str += tree_header; + str += "\n\n"; + + // Fill in the functions for each automaton + for(int i = 0; i < num_components; i++) { + str += "ap_uint<1> automata_" + std::to_string(i) + "(uint8_t input);\n"; + } + str += "\n"; + str += "#endif"; + + // Write header to file 'automata.hpp' + writeStringToFile(str, "automata.hpp"); +} + +string generateHTree(int num_automata, int split_factor) { + // Do we want to balance partitioning by # states? + // For now we're gonna go the lazy route + string str = ""; + string header_str = "// ROOT (LEVEL 0)\n"; + + str += "////////////////////////////////////////\n"; + str += "// Copyright goes here\n"; + str += "// This HLS was emitted by VASim\n"; + str += "////////////////////////////////////////\n"; + str += "\n"; + + // Print headers + str += "#include \"automata.hpp\"\n"; + str += "\n"; + + // Some details for debugging + str += "// This file was generated for " + std::to_string(num_automata) + " automata.\n"; + str += "// with split factor " + std::to_string(split_factor) + "\n\n"; + + // Root function automata(input) + str += "automata_output automata(uint8_t input){\n"; + header_str += "automata_output automata(uint8_t input);\n\n"; + str += "\t#pragma HLS INLINE OFF\n\n"; + + // We assume that we have more than split_factor automata + assert(num_automata > split_factor); + + // This is how we'll pass ranges + struct range { + int range[2]; + std::string name; + }; + + // Start breaking the automata up recursively + std::queue q; + int split_size = num_automata / split_factor; + int left_overs = num_automata % split_factor; + + std::cout << "Num automata: " << num_automata << std::endl; + std::cout << "Split Size: " << split_size << std::endl; + std::cout << "Left Overs: " << left_overs << std::endl; + + // The root function has a report = OR(all automata) + str += "\tstatic uint8_t report = "; + for(int i = 0; i < split_factor; i++) { + string automata_branch = "automata_tree_" + std::to_string(i); + str += automata_branch + "(input)"; + + // If not the last branch for this level, then | + if(i != (split_factor - 1)) { + str += " | "; + } + + range tmp; + tmp.range[0] = split_size * i; + tmp.range[1] = split_size * i + split_size - 1; + tmp.name = automata_branch; + q.push(tmp); + } + str += ";\n\n"; + str += "\treturn report;\n"; + str += "}\n\n"; + + // Now lets start processing the branches / leaves + + std::cout << "Contents of the Queue" << std::endl; + while (!q.empty()) { + range front = q.front(); + int automata_left = (front.range[1] - front.range[0] + 1); + split_size = automata_left / split_factor; + left_overs = automata_left % split_factor; + std::cout << "\t Range:[" << front.range[0] << ", " << front.range[1] << "], Left: " << automata_left << " Name:" << front.name << std::endl; + + // Check if leaf + if(automata_left <= split_factor){ + str += "uint8_t " + front.name + "(uint8_t input){\n"; + header_str += "uint8_t " + front.name + "(uint8_t input);\n"; + str += "\t#pragma HLS INLINE OFF\n\n"; + str += "\tstatic uint8_t report = "; + std::cout <<"Found a LEAF!!: " << front.range[0] << "-" << front.range[1] << std::endl; + for(int i = front.range[0]; i <= front.range[1]; i++){ + str += "automata_" + std::to_string(i) + "(input)"; + if(i != front.range[1]) { + str += " | "; + } + } + str += ";\n\n"; + str += "\treturn report;\n"; + str += "}\n\n"; + } + // Generate name of function based on depth + else { + str += "uint8_t " + front.name + "(uint8_t input){\n"; + header_str += "uint8_t " + front.name + "(uint8_t input);\n"; + str += "\t#pragma HLS INLINE OFF\n\n"; + str += "\tstatic uint8_t report = "; + + for(int i = 0; i < split_factor; i++){ + string new_name = front.name + "_" + std::to_string(i); + str += new_name + "(input)"; + if(i != (split_factor - 1)){ + str += " | "; + } + range tmp; + tmp.range[0] = front.range[0] + (split_size * i); + tmp.range[1] = front.range[0] + (split_size * i + split_size - 1); + tmp.name = new_name; + q.push(tmp); + } + str += ";\n\n"; + str += "\treturn report;\n"; + str += "}\n\n"; + } + + q.pop(); + } + + writeStringToFile(str, "automata_tree.cpp"); + + return header_str; +} + +bool sort_automata_by_states (Automata* i,Automata* j) { + return (i->getElements().size() < j->getElements().size()); +} + +// Function for automataToHLSFiles to generate ranges of values for symbol sets +vector > getRanges(vector symbolSet){ + + // Sort the symbols + std::sort (symbolSet.begin(), symbolSet.end()); + + // We're going to return pairs representing continuous numerical ranges + vector > ranges; + + int first, second; + + for(int i = 0; i < symbolSet.size(); i++){ + int current = symbolSet[i]; + + if(i == 0){ // Start + first = current; + second = first; + } + // If contiguous + else if(current == (second + 1)){ + second = current; + } + // Not contiguous + else { + pair temp(first, second); + ranges.push_back(temp); + + first = current; + second = current; + } + } + + pair temp(first, second); + ranges.push_back(temp); + return ranges; +} + +void Automata::automataToHLSFiles(int N, int split_factor) { + + bool or_all = false; + bool one_function_per_file = false; + bool inlined = false; + bool single_file = true; + bool sort_automata = false; + + // This is representing character sets in the AP way + // They are stored in memories and evaluated at runtime; this is SLOW + bool bitwise = false; + + vector connected_components = splitConnectedComponents(); + + if(sort_automata){ + std::sort (connected_components.begin(), connected_components.end(), sort_automata_by_states); + } + + int index = 0; + for(auto cc: connected_components) + std::cout << "Automata " << index++ << " size: " << std::to_string(cc->getElements().size()) << std::endl; + + assert(N <= connected_components.size()); + + Automata first = Automata(*connected_components[0]); + + // Chooses the first N components + vector subset; + for (int i = 0; i < N; i++) { + subset.push_back(connected_components[i]); + first.unsafeMerge(connected_components[i]); + } + first.finalizeAutomata(); + + // Drop the N components into an Automata file + first.automataToANMLFile("Automata.anml"); + + // Update num_components + int num_components = subset.size(); + + std::cout << "Splitting automata into " << num_components << " components" << std::endl; + std::cout << "Where each component is represented by a separate HLS function" << std::endl; + + // Set the return type; one bit per automaton + // TODO: we may have multiple report states in the future + string return_type = "ap_uint<" + std::to_string(N) + ">"; + + // Generate kernel call string + string str = ""; + str += ""; + + int i = 0; + // For each automata component in the graph + for(auto aut : subset){ + string automata_name = "automata_" + std::to_string(i); + + if(!single_file) + str = ""; + + if(!single_file || i == 0){ + // Print copyright + str += "////////////////////////////////////////\n"; + str += "// Copyright goes here\n"; + str += "// This HLS was emitted by VASim\n"; + str += "////////////////////////////////////////\n"; + str += "\n"; + } + string report_string = "return ("; + + // Print headers + if(!single_file || i == 0){ + str += "#include \"automata.hpp\"\n"; + } + + str += "\n"; + + // Print function + // TO DO: Future automata may have multiple return states + str += "ap_uint<1> " + automata_name + "(uint8_t input) {\n"; + + if(!inlined) + str += "\t#pragma HLS INLINE OFF\n"; + + // Add pipeline pragma to make sure automaton finishes in one cycle! + str += "\t#pragma HLS pipeline II=1\n"; + + str += "\n"; + + // Stamp out states + str += "\t//States\n"; + str += "\tstatic uint8_t input_r = 0;\n"; + str += "\tstatic const ap_uint<1> start_state = 1;\n"; + + // For each state... + for(auto e : aut->getElements()){ + Element *el = e.second; + + // TODO: In the future support counters and gates + if(el->isSpecialElement()){ + std::cout << "Element " << el->getId() + " is not an STE; FAIL\n"; + exit(-1); + } + STE *s = static_cast(el); + string s_id = s->getId(); + char start = (s->isStart()) ? '1' : '0'; + vector integerSymbolSet = s->getIntegerSymbolSet(); + + // Create state register for STE + str += "\tstatic ap_uint<1> state_" + s_id + "_enable = " + start + ";\n"; + + // Create symbol set array for STE + if(bitwise) { + str += "\tconst uint8_t state_" + s_id + "_char[" + std::to_string(integerSymbolSet.size()) + "] = {"; + // Populate symbol set array with symbol set of STE + for(int i = 0; i < integerSymbolSet.size(); i++) { + str += std::to_string(integerSymbolSet[i]); + if(i != (integerSymbolSet.size() - 1)){ + str += ","; + } + } + str += "};\n"; + str += "\n"; + } + } + + // Add State Logic + str += "\t // State Logic\n"; + + for(auto e : aut->getElements()){ + Element *el = e.second; + + // TODO: For now only support NFA states; future add counters and OR/AND gates + if(el->isSpecialElement()){ + std::cout << "Element " << el->getId() + " is not an STE; FAIL\n"; + exit(-1); + } + + STE *s = static_cast(el); + string s_id = s->getId(); + vector integerSymbolSet = s->getIntegerSymbolSet(); + + str += "\tap_uint<1> ste_" + s_id + " = (state_" + s_id + "_enable) &&\n"; + str += "\t\t("; + + // This is a * state + if(bitwise){ + if(integerSymbolSet.size() == 256) { + str += "1"; + } + else { + // There is room to make this more concise + for(int i = 0; i < integerSymbolSet.size(); i++){ + str += "(input_r == state_" + s_id + "_char[" + std::to_string(i) + "])"; + if(i != integerSymbolSet.size() - 1){ + str += " || \n"; + } + } + } + } + else{ + vector> ranges = getRanges(integerSymbolSet); + for(int i = 0; i < ranges.size(); i++){ + pair pair = ranges[i]; + if(pair.first == pair.second){ + str += "(input_r == " + std::to_string(pair.first) + ")"; + } + else{ + str += "(input_r >= " + std::to_string(pair.first) + " && input_r <= " + std::to_string(pair.second) + ")"; + } + if(i != (ranges.size() - 1)){ + str += "|| "; + } + } + + } + str += ");\n"; + str += "\n"; + } + + // Register updates (edges) + str += "\t// Edges\n"; + str += "\tinput_r = input;\n"; + + bool first = true; + for(auto e: aut->getElements()){ + Element *el = e.second; + if(el->isSpecialElement()){ + std::cout << "Element " << el->getId() + " is not an STE; FAIL\n"; + exit(-1); + } + STE *s = static_cast(el); + string s_id = s->getId(); + + if(s->isReporting()){ + if(!first){ + report_string += " || "; + } + else{ + first = false; + } + report_string += "ste_" + s_id; + } + + str += "\tstate_" + s_id + "_enable = "; + if(s->isStart()) { // For now we only support all-data + str += "start_state;\n"; // Every cycle this state is enabled + } + else { + str += "("; + auto in_edges = s->getInputs(); + bool first = true; + for(auto in : in_edges){ + if(!first){ + str += " || "; + } + else { + first = false; + } + str += "ste_" + in.first; + } + str += ");\n"; + } + } + + report_string += ");"; + str += "\t" + report_string + "\n"; + str += "}\n\n"; + + if(!single_file) + writeStringToFile(str, "automata_" + std::to_string(i) + ".cpp"); + i++; + } + if(single_file) + writeStringToFile(str, "automata_single_file.cpp"); + + string tree_header = ""; + + // What should this threashold me? + if(N > 512){ + tree_header += generateHTree(N, split_factor); + } + + writeHeaderFile(num_components, return_type, tree_header); +} + /** * Outputs automata to Verilog HDL description following the algorithm originally developed by Xiaoping Huang and Mohamed El-Hadedy. */ @@ -2503,7 +2940,7 @@ void Automata::specialElementSimulation2() { if(result){ spel->enableChildSTEs(&enabledSTEs); spel->enableChildSpecialElements(&enabledSpecialElements); - } + } } } @@ -2532,7 +2969,7 @@ void Automata::specialElementSimulation() { queued[e.second->getIntId()] = false; } - // fill work_q with specel children of STEs + // fill work_q with special children of STEs for(auto e : elements) { if(!e.second->isSpecialElement()){ diff --git a/src/main.cpp b/src/main.cpp index e6262d1..2785875 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -33,8 +33,10 @@ void usage(char * argv) { printf(" -n, --nfa Output automata as nfa readable by Michela Becchi's tools\n"); printf(" -D, --dfa Convert automata to DFA\n"); printf(" -f, --hdl Output automata as one-hot encoded verilog HDL for execution on an FPGA (EXPERIMENTAL)\n"); + printf(" -F, --hls Output automata as VITIS HLS-compatible C++ (EXPERIMENTAL - only supports STEs) Provide number of automata\n"); printf(" -B, --blif Output automata as .blif circuit for place-and-route using VPR.\n"); printf(" --graph Output automata as .graph file for HyperScan.\n"); + printf(" -S, --split Specify number of connected components in separate files.\n"); printf("\n OPTIMIZATIONS:\n"); printf(" -O, --optimize-global Run all optimizations on all automata subgraphs.\n"); @@ -45,7 +47,7 @@ void usage(char * argv) { printf(" --enforce-fanin= Enforces a fan-in limit, replicating nodes until no node has a fan-in of larger than .\n"); printf(" --enforce-fanout= Enforces a fan-out limit, replicating nodes until no node has a fan-out of larger than .\n"); printf(" --widen Pads each state with a zero state for patterns where the input is 16 bits (common in YARA rules).\n"); - printf(" --2-stride Two strides automata if possible.\n"); + printf(" --2-stride Two strides automata if possible.\n"); printf("\n MULTITHREADING:\n"); printf(" -T, --threads Specify number of threads to compute connected components of automata\n"); @@ -91,7 +93,9 @@ int main(int argc, char * argv[]) { bool to_nfa = false; bool to_dfa = false; bool to_hdl = false; + bool to_hls = false; bool to_blif = false; + bool cc = false; uint32_t num_threads = 1; uint32_t num_threads_packets = 1; bool to_graph = false; @@ -101,6 +105,9 @@ int main(int argc, char * argv[]) { uint32_t dump_state_cycle = 0; bool widen = false; bool two_stride = false; + bool split = false; + uint32_t split_size = 0; + uint32_t num_automata = 0; // long option switches const int32_t graph_switch = 1000; @@ -111,7 +118,7 @@ int main(int argc, char * argv[]) { const int32_t two_stride_switch = 1005; int c; - const char * short_opt = "thsqrbnfcdBDeamxipOLT:P:"; + const char * short_opt = "thsqrbnfcdBDeamxipF:OLT:P:S:"; struct option long_opt[] = { {"help", no_argument, NULL, 'h'}, @@ -125,6 +132,7 @@ int main(int argc, char * argv[]) { {"nfa", no_argument, NULL, 'n'}, {"dfa", no_argument, NULL, 'D'}, {"hdl", no_argument, NULL, 'f'}, + {"hls", no_argument, NULL, 'F'}, {"blif", no_argument, NULL, 'B'}, {"profile", no_argument, NULL, 'p'}, {"charset", no_argument, NULL, 'c'}, @@ -134,6 +142,7 @@ int main(int argc, char * argv[]) { {"remove-ors", no_argument, NULL, 'x'}, {"thread-width", required_argument, NULL, 'T'}, {"thread-height", required_argument, NULL, 'P'}, + {"split", required_argument, NULL, 'S'}, {"graph", no_argument, NULL, graph_switch}, {"enforce-fanin", required_argument, NULL, fanin_switch}, {"enforce-fanout", required_argument, NULL, fanout_switch}, @@ -218,6 +227,11 @@ int main(int argc, char * argv[]) { num_threads_packets = atoi(optarg); break; + case 'S': + split = true; + split_size = atoi(optarg); + break; + case 'D': to_dfa = true; break; @@ -229,10 +243,19 @@ int main(int argc, char * argv[]) { case 'f': to_hdl = true; break; + + case 'F': + to_hls = true; + num_automata = atoi(optarg); + break; case 'B': to_blif = true; break; + + case 'C': + cc = true; + break; case 'h': usage(argv[0]); @@ -386,9 +409,15 @@ int main(int argc, char * argv[]) { // Partition automata into connected components if(!quiet) cout << "Finding connected components..." << endl; + vector ccs; - ccs = ap.splitConnectedComponents(); - + ccs = ap.splitConnectedComponents(); + + std::cout << "Full Automata " << std::to_string(ap.getElements().size()) << std::endl; + + int index = 0; + for(auto cc: ccs) + std::cout << "Automata " << index++ << " size: " << std::to_string(cc->getElements().size()) << std::endl; if(!quiet) cout << endl; @@ -421,10 +450,21 @@ int main(int argc, char * argv[]) { counter++; } + // std::cout << "Post merging" << std::endl; + // index = 0; + // for(auto cc: ccs) + // std::cout << "Automata " << index++ << " size: " << std::to_string(cc->getElements().size()) << std::endl; + + // finalize copied automata for(Automata *a : merged) { a->finalizeAutomata(); } + + // std::cout << "Post finalizing" << std::endl; + // index = 0; + // for(auto cc: ccs) + // std::cout << "Automata " << index++ << " size: " << std::to_string(cc->getElements().size()) << std::endl; if(!quiet) cout << endl; @@ -433,7 +473,6 @@ int main(int argc, char * argv[]) { // Set up multi-dimensional structure for parallelization Automata *automata[num_threads][num_threads_packets]; - if(optimize_local) { if(!quiet){ @@ -444,7 +483,7 @@ int main(int argc, char * argv[]) { } counter = 0; - for(Automata *a : merged) { + for(Automata *a : merged) { /********************* * LOCAL OPTIMIZATIONS *********************/ @@ -538,6 +577,36 @@ int main(int argc, char * argv[]) { a->automataToHDLFile("automata_" + to_string(counter) + ".v"); } + // Emit as HLS + if(to_hls) { + int split_factor = 5; + a->automataToHLSFiles(num_automata, split_factor); + } + + // Emit as split components + if(split) { + + int num_components = ccs.size(); + int number_files = num_components / split_size; + int left_overs = num_components % split_size; + + for(int i = 0; i < number_files; i++){ + Automata first = Automata(*ccs[i * split_size]); + for(int j = 0; j < split_size; j++) + first.unsafeMerge(ccs[i * split_size + j]); + first.finalizeAutomata(); + first.automataToANMLFile("automata_split_" + std::to_string(i) + ".anml"); + } + + if(left_overs){ + Automata first = Automata(*ccs[number_files * split_size]); + for(int i = number_files * split_size; i < num_components; i++) + first.unsafeMerge(ccs[i]); + first.finalizeAutomata(); + first.automataToANMLFile("automata_split_" + std::to_string(number_files) + ".anml"); + } + } + // Emit as .blif circuit file if(to_blif){ if(!quiet) From 201f5eb5b981a2d4b902291aeec5ea3400e257bd Mon Sep 17 00:00:00 2001 From: Tommy Tracy II Date: Wed, 28 Feb 2024 15:21:50 -0500 Subject: [PATCH 8/8] Resolved merge issues with --hls and --split --- src/main.cpp | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 2347e8b..d68512d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -108,8 +108,8 @@ int main(int argc, char * argv[]) { bool widen = false; bool two_stride = false; bool split = false; - uint32_t split_size = 0; uint32_t num_automata = 0; + uint32_t split_count = 0; // long option switches const int32_t graph_switch = 1000; @@ -231,7 +231,7 @@ int main(int argc, char * argv[]) { case 'S': split = true; - split_size = atoi(optarg); + split_count = atoi(optarg); break; case 'D': @@ -588,24 +588,26 @@ int main(int argc, char * argv[]) { // Emit as split components if(split) { - int num_components = ccs.size(); - int number_files = num_components / split_size; - int left_overs = num_components % split_size; + vector components = ap.splitConnectedComponents(); - for(int i = 0; i < number_files; i++){ - Automata first = Automata(*ccs[i * split_size]); - for(int j = 0; j < split_size; j++) - first.unsafeMerge(ccs[i * split_size + j]); - first.finalizeAutomata(); - first.automataToANMLFile("automata_split_" + std::to_string(i) + ".anml"); - } + int num_components = components.size(); + int num_automata_per_file = num_components / split_count; - if(left_overs){ - Automata first = Automata(*ccs[number_files * split_size]); - for(int i = number_files * split_size; i < num_components; i++) - first.unsafeMerge(ccs[i]); + for(int i = 0; i < split_count; i++){ + Automata first = Automata(*components[i * num_automata_per_file]); + for(int j = 0; j < num_automata_per_file; j++) + first.unsafeMerge(components[i * num_automata_per_file + j]); + if(i == (split_count - 1)){ + // Put the remaining automata in the last split file + for(int j = split_count * num_automata_per_file; j < num_components; j++){ + first.unsafeMerge(components[j]); + } + } first.finalizeAutomata(); - first.automataToANMLFile("automata_split_" + std::to_string(number_files) + ".anml"); + if(to_mnrl) + first.automataToMNRLFile("automata_split_" + to_string(i) + ".mnrl"); + else + first.automataToANMLFile("automata_split_" + to_string(i) + ".anml"); } }