Fix for detecting left recursion in rules like:

README.md

@@ -3,3 +3,23 @@
 Coco/R is a compiler generator, which takes an attributed grammar of a source language and generates a scanner and a parser for this language. The scanner works as a deterministic finite automaton. The parser uses recursive descent. LL(1) conflicts can be resolved by a multi-symbol lookahead or by semantic checks. Thus the class of accepted grammars is LL(k) for an arbitrary k.
 
 http://ssw.jku.at/coco/
+
+And this are my main modifications to the original:
+
+- Enhance left recursion detection
+
+- Allow semantic actions on `token declaration` similar to `pragmas` but the code executes on the Scanner
+
+- Allow till 8 characters as comment delimiters
+
+- Add option `-genRREBNF` to generate an EBNF grammar to crate railroad diagrams at https://www.bottlecaps.de/rr/ui
+
+- Add option `-geAST` to generate code to generate `parser syntax tree` based on https://github.com/rochus-keller/EbnfStudio
+
+- Add option `-ignoreGammarErrors` to make easier to develop grammars, like commenting one non terminal and still generating the parser and scanner even with sevral non reachable non terminals
+
+- Add a `TERMINALS` section to generate user define tokens not managed by the Scanner (from cocoxml)
+
+- Generate between comments the correspondent representation of several magic numbers (mainly Tokens) 
+
+See also https://github.com/mingodad/CocoR-CPP and https://github.com/mingodad/CocoR-CSharp

src/Coco.cs

@@ -4,24 +4,24 @@
 extended by M. Loeberbauer & A. Woess, Univ. of Linz
 with improvements by Pat Terry, Rhodes University
 
-This program is free software; you can redistribute it and/or modify it 
-under the terms of the GNU General Public License as published by the 
-Free Software Foundation; either version 2, or (at your option) any 
+This program is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2, or (at your option) any
 later version.
 
-This program is distributed in the hope that it will be useful, but 
-WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 
-or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License 
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 for more details.
 
-You should have received a copy of the GNU General Public License along 
-with this program; if not, write to the Free Software Foundation, Inc., 
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 
 As an exception, it is allowed to write an extension of Coco/R that is
 used as a plugin in non-free software.
 
-If not otherwise stated, any source code generated by Coco/R (other than 
+If not otherwise stated, any source code generated by Coco/R (other than
 Coco/R itself) does not fall under the GNU General Public License.
 -------------------------------------------------------------------------*/
 /*-------------------------------------------------------------------------
@@ -34,7 +34,7 @@ Trace output options
   6 | S: prints the symbol table (terminals, nonterminals, pragmas)
   7 | X: prints a cross reference list of all syntax symbols
   8 | P: prints statistics about the Coco run
-  
+
   Trace output can be switched on by the pragma
     $ { digit | letter }
   in the attributed grammar or as a command-line option
@@ -46,31 +46,36 @@ in the attributed grammar or as a command-line option
 namespace at.jku.ssw.Coco {
 
 public class Coco {
-		
+
 	public static int Main (string[] arg) {
 		Console.WriteLine("Coco/R (Apr 19, 2011)");
 		string srcName = null, nsName = null, frameDir = null, ddtString = null,
 		traceFileName = null, outDir = null;
-		bool emitLines = false;
+		bool emitLines = false, ignoreErrors = false, genAST = false, genRREBNF = false;
 		int retVal = 1;
 		for (int i = 0; i < arg.Length; i++) {
 			if (arg[i] == "-namespace" && i < arg.Length - 1) nsName = arg[++i].Trim();
 			else if (arg[i] == "-frames" && i < arg.Length - 1) frameDir = arg[++i].Trim();
 			else if (arg[i] == "-trace" && i < arg.Length - 1) ddtString = arg[++i].Trim();
 			else if (arg[i] == "-o" && i < arg.Length - 1) outDir = arg[++i].Trim();
 			else if (arg[i] == "-lines") emitLines = true;
+			else if (arg[i] == "-genRREBNF") genRREBNF = true;
+			else if (arg[i] == "-genAST") genAST = true;
+			else if (arg[i] == "-ignoreErrors") ignoreErrors = true;
 			else srcName = arg[i];
 		}
 		if (arg.Length > 0 && srcName != null) {
 			try {
 				string srcDir = Path.GetDirectoryName(srcName);
-				
+
 				Scanner scanner = new Scanner(srcName);
 				Parser parser = new Parser(scanner);
 
 				traceFileName = Path.Combine(srcDir, "trace.txt");
 				parser.trace = new StreamWriter(new FileStream(traceFileName, FileMode.Create));
 				parser.tab = new Tab(parser);
+				if(genAST) parser.tab.genAST = true;
+				if(ignoreErrors) parser.tab.ignoreErrors = true;
 				parser.dfa = new DFA(parser);
 				parser.pgen = new ParserGen(parser);
 
@@ -80,6 +85,9 @@ public static int Main (string[] arg) {
 				parser.tab.frameDir = frameDir;
 				parser.tab.outDir = (outDir != null) ? outDir : srcDir;
 				parser.tab.emitLines = emitLines;
+				parser.tab.genRREBNF = genRREBNF;
+				parser.tab.genAST = genAST;
+				parser.tab.ignoreErrors = ignoreErrors;
 				if (ddtString != null) parser.tab.SetDDT(ddtString);
 
 				parser.Parse();
@@ -88,7 +96,7 @@ public static int Main (string[] arg) {
 				FileInfo f = new FileInfo(traceFileName);
 				if (f.Length == 0) f.Delete();
 				else Console.WriteLine("trace output is in " + traceFileName);
-				Console.WriteLine("{0} errors detected", parser.errors.count);
+				Console.WriteLine("{0} error(s) detected", parser.errors.count);
 				if (parser.errors.count == 0) { retVal = 0; }
 			} catch (IOException) {
 				Console.WriteLine("-- could not open " + traceFileName);
@@ -103,6 +111,9 @@ public static int Main (string[] arg) {
 			                  + "  -trace     <traceString>{0}"
 			                  + "  -o         <outputDirectory>{0}"
 			                  + "  -lines{0}"
+			                  + "  -genRREBNF{0}"
+			                  + "  -genAST{0}"
+			                  + "  -ignoreErrors ignore grammar errors for developing purposes{0}"
 			                  + "Valid characters in the trace string:{0}"
 			                  + "  A  trace automaton{0}"
 			                  + "  F  list first/follow sets{0}"
@@ -118,7 +129,7 @@ public static int Main (string[] arg) {
 		}
 		return retVal;
 	}
-		
+
 } // end Coco
 
 } // end namespace

src/Copyright.frame

@@ -0,0 +1,26 @@
+/*----------------------------------------------------------------------
+Compiler Generator Coco/R,
+Copyright (c) 1990, 2004 Hanspeter Moessenboeck, University of Linz
+extended by M. Loeberbauer & A. Woess, Univ. of Linz
+with improvements by Pat Terry, Rhodes University
+
+This program is free software; you can redistribute it and/or modify it
+under the terms of the GNU General Public License as published by the
+Free Software Foundation; either version 2, or (at your option) any
+later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
+or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License along
+with this program; if not, write to the Free Software Foundation, Inc.,
+59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+
+As an exception, it is allowed to write an extension of Coco/R that is
+used as a plugin in non-free software.
+
+If not otherwise stated, any source code generated by Coco/R (other than
+Coco/R itself) does not fall under the GNU General Public License.
+-----------------------------------------------------------------------*/

src/DFA.cs

@@ -177,10 +177,30 @@ public class Range {
 		public int from, to;
 		public Range next;
 		public Range(int from, int to) { this.from = from; this.to = to; }
+
+		public override string ToString() {
+			if (from == to)
+				return from.ToString("X");
+			if (from <= 256 && to <= 256)
+				return string.Format("{0:X2}-{1:X2}", from, to);
+			return string.Format("{0:X4}-{1:X4}", from, to);
+		}
 	}
 
 	public Range head;
 
+	public override string ToString() {
+		if (head == null) return "[]";
+		StringBuilder sb = new StringBuilder();
+		sb.Append('[');
+		for (Range cur = head; cur != null; cur = cur.next) {
+			if (cur != head) sb.Append('|');
+			sb.Append(cur.ToString());
+		}
+		sb.Append(']');
+		return sb.ToString();
+	}
+
 	public bool this[int i] {
 		get {
 			for (Range p = head; p != null; p = p.next)
@@ -284,7 +304,7 @@ public void Fill() {
 //-----------------------------------------------------------------------------
 //  Generator
 //-----------------------------------------------------------------------------
-class Generator {
+public class Generator {
 	private const int EOF = -1;
 
 	private FileStream fram;
@@ -478,7 +498,7 @@ void DeleteRedundantStates() {
 		for (State s1 = firstState.next; s1 != null; s1 = s1.next) // firstState cannot be final
 			if (used[s1.nr] && s1.endOf != null && s1.firstAction == null && !s1.ctx)
 				for (State s2 = s1.next; s2 != null; s2 = s2.next)
-					if (used[s2.nr] && s1.endOf == s2.endOf && s2.firstAction == null & !s2.ctx) {
+					if (used[s2.nr] && s1.endOf == s2.endOf && s2.firstAction == null && !s2.ctx) {
 						used[s2.nr] = false; newState[s2.nr] = s1;
 					}
 		for (State state = firstState; state != null; state = state.next)
@@ -831,8 +851,8 @@ string CommentStr(Node p) {
 			} else parser.SemErr("comment delimiters may not be structured");
 			p = p.next;
 		}
-		if (s.Length == 0 || s.Length > 2) {
-			parser.SemErr("comment delimiters must be 1 or 2 characters long");
+		if (s.Length == 0 || s.Length > 8) {
+			parser.SemErr("comment delimiters must be between 1 to 8 characters long");
 			s = new StringBuilder("?");
 		}
 		return s.ToString();
@@ -846,30 +866,45 @@ public void NewComment(Node from, Node to, bool nested) {
 
 //------------------------ scanner generation ----------------------
 
+	void GenCommentIndented(int n, string s) {
+		for(int i= 1; i < n; ++i) gen.Write("\t");
+		gen.Write(s);
+	}
+
 	void GenComBody(Comment com) {
+		int imax = com.start.Length-1;
 		gen.WriteLine(  "\t\t\tfor(;;) {");
 		gen.Write    (  "\t\t\t\tif ({0}) ", ChCond(com.stop[0])); gen.WriteLine("{");
 		if (com.stop.Length == 1) {
 			gen.WriteLine("\t\t\t\t\tlevel--;");
 			gen.WriteLine("\t\t\t\t\tif (level == 0) { oldEols = line - line0; NextCh(); return true; }");
 			gen.WriteLine("\t\t\t\t\tNextCh();");
 		} else {
-			gen.WriteLine("\t\t\t\t\tNextCh();");
-			gen.WriteLine("\t\t\t\t\tif ({0}) {{", ChCond(com.stop[1]));
+			for(int sidx = 1; sidx <= imax; ++sidx) {
+				gen.WriteLine("\t\t\t\t\tNextCh();");
+				gen.WriteLine("\t\t\t\t\tif ({0}) {{", ChCond(com.stop[sidx]));
+			}
 			gen.WriteLine("\t\t\t\t\t\tlevel--;");
-			gen.WriteLine("\t\t\t\t\t\tif (level == 0) { oldEols = line - line0; NextCh(); return true; }");
+			gen.WriteLine("\t\t\t\t\t\tif (level == 0) { /*oldEols = line - line0;*/ NextCh(); return true; }");
 			gen.WriteLine("\t\t\t\t\t\tNextCh();");
-			gen.WriteLine("\t\t\t\t\t}");
+			for(int sidx = imax; sidx > 0; --sidx) {
+				gen.WriteLine("\t\t\t\t\t}");
+			}
 		}
 		if (com.nested) {
 			gen.Write    ("\t\t\t\t}"); gen.Write(" else if ({0}) ", ChCond(com.start[0])); gen.WriteLine("{");
 			if (com.start.Length == 1)
 				gen.WriteLine("\t\t\t\t\tlevel++; NextCh();");
 			else {
-				gen.WriteLine("\t\t\t\t\tNextCh();");
-				gen.Write    ("\t\t\t\t\tif ({0}) ", ChCond(com.start[1])); gen.WriteLine("{");
+				int imaxN = com.start.Length-1;
+				for(int sidx = 1; sidx <= imaxN; ++sidx) {
+					gen.WriteLine("\t\t\t\t\tNextCh();");
+					gen.Write    ("\t\t\t\t\tif ({0}) ", ChCond(com.start[sidx])); gen.WriteLine("{");
+				}
 				gen.WriteLine("\t\t\t\t\t\tlevel++; NextCh();");
-				gen.WriteLine("\t\t\t\t\t}");
+				for(int sidx = imaxN; sidx > 0; --sidx) {
+					gen.WriteLine("\t\t\t\t\t}");
+				}
 			}
 		}
 		gen.WriteLine(    "\t\t\t\t} else if (ch == Buffer.EOF) return false;");
@@ -881,17 +916,20 @@ void GenComment(Comment com, int i) {
 		gen.WriteLine();
 		gen.Write    ("\tbool Comment{0}() ", i); gen.WriteLine("{");
 		gen.WriteLine("\t\tint level = 1, pos0 = pos, line0 = line, col0 = col, charPos0 = charPos;");
+		gen.WriteLine("\t\tNextCh();");
 		if (com.start.Length == 1) {
-			gen.WriteLine("\t\tNextCh();");
 			GenComBody(com);
 		} else {
-			gen.WriteLine("\t\tNextCh();");
-			gen.Write    ("\t\tif ({0}) ", ChCond(com.start[1])); gen.WriteLine("{");
-			gen.WriteLine("\t\t\tNextCh();");
+			int imax = com.start.Length-1;
+			for(int sidx = 1; sidx <= imax; ++sidx) {
+				gen.Write    ("\t\tif ({0}) ", ChCond(com.start[sidx])); gen.WriteLine("{");
+				gen.WriteLine("\t\t\tNextCh();");
+			}
 			GenComBody(com);
-			gen.WriteLine("\t\t} else {");
-			gen.WriteLine("\t\t\tbuffer.Pos = pos0; NextCh(); line = line0; col = col0; charPos = charPos0;");
-			gen.WriteLine("\t\t}");
+			for(int sidx = imax; sidx > 0; --sidx) {
+				gen.WriteLine("\t\t}");
+			}
+			gen.WriteLine("\t\tbuffer.Pos = pos0; NextCh(); line = line0; col = col0; charPos = charPos0;");
 			gen.WriteLine("\t\treturn false;");
 		}
 		gen.WriteLine("\t}");
@@ -924,12 +962,12 @@ void GenLiterals () {
 		gen.WriteLine("\t\t\tdefault: break;");
 		gen.Write("\t\t}");
 	}
-	
+
 	void WriteState(State state) {
 		Symbol endOf = state.endOf;
 		gen.WriteLine("\t\t\tcase {0}:", state.nr);
 		if (endOf != null && state.firstAction != null) {
-			gen.WriteLine("\t\t\t\trecEnd = pos; recKind = {0};", endOf.n);
+			gen.WriteLine("\t\t\t\trecEnd = pos; recKind = {0} /* {1} */;", endOf.n, endOf.name);
 		}
 		bool ctxEnd = state.ctx;
 		for (Action action = state.firstAction; action != null; action = action.next) {
@@ -958,10 +996,15 @@ void WriteState(State state) {
 		if (endOf == null) {
 			gen.WriteLine("goto case 0;}");
 		} else {
-			gen.Write("t.kind = {0}; ", endOf.n);
+			gen.Write("t.kind = {0} /* {1} */; ", endOf.n, endOf.name);
 			if (endOf.tokenKind == Symbol.classLitToken) {
 				gen.WriteLine("t.val = new String(tval, 0, tlen); CheckLiteral(); return t;}");
 			} else {
+				if(endOf.semPos != null && endOf.typ == Node.t) {
+					gen.Write(" {");
+					parser.pgen.CopySourcePart(parser, gen, endOf.semPos, 0);
+					gen.Write("};");
+				}
 				gen.WriteLine("break;}");
 			}
 		}
@@ -1023,20 +1066,21 @@ public void WriteScanner() {
 		}
 		g.CopyFramePart("-->literals"); GenLiterals();
 		g.CopyFramePart("-->scan1");
-		gen.Write("\t\t\t");
+		gen.Write("\t\t\t\t");
 		if (tab.ignored.Elements() > 0) { PutRange(tab.ignored); } else { gen.Write("false"); }
 		g.CopyFramePart("-->scan2");
 		if (firstComment != null) {
-			gen.Write("\t\tif (");
+			gen.Write("\t\t\tif (");
 			com = firstComment; comIdx = 0;
 			while (com != null) {
 				gen.Write(ChCond(com.start[0]));
 				gen.Write(" && Comment{0}()", comIdx);
 				if (com.next != null) gen.Write(" ||");
 				com = com.next; comIdx++;
 			}
-			gen.Write(") return NextToken();");
+			gen.Write(") continue;");
 		}
+		g.CopyFramePart("-->scan22");
 		if (hasCtxMoves) { gen.WriteLine(); gen.Write("\t\tint apx = 0;"); } /* pdt */
 		g.CopyFramePart("-->scan3");
 		for (State state = firstState.next; state != null; state = state.next)