diff --git a/.gitignore b/.gitignore index 23616f6..68f5431 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ -################################################################################ -# This .gitignore file was automatically created by Microsoft(R) Visual Studio. -################################################################################ -ColorzCore/obj -ColorzCore/.vs +obj/ +bin/ +.vs/ *.cache diff --git a/ColorzCore/App.config b/ColorzCore/App.config deleted file mode 100644 index d740e88..0000000 --- a/ColorzCore/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ColorzCore/ColorzCore.Framework.csproj b/ColorzCore/ColorzCore.Framework.csproj new file mode 100644 index 0000000..fc5ed33 --- /dev/null +++ b/ColorzCore/ColorzCore.Framework.csproj @@ -0,0 +1,10 @@ + + + net48 + Exe + annotations + ColorzCore + 9 + bin/Framework + + diff --git a/ColorzCore/ColorzCore.csproj b/ColorzCore/ColorzCore.csproj index 842e13f..11a245f 100644 --- a/ColorzCore/ColorzCore.csproj +++ b/ColorzCore/ColorzCore.csproj @@ -1,124 +1,9 @@ - - - + - Debug - AnyCPU - {B98F7CCF-9CAA-406E-88D7-2040FA99F631} + net6.0 Exe - ColorzCore - ColorzCore - v4.5.2 - 512 - true + enable + 9 + bin/Core - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - bin\32bit\ - x86 - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ColorzCore/ColorzCore.sln b/ColorzCore/ColorzCore.sln index f0a2e0f..805d6ec 100644 --- a/ColorzCore/ColorzCore.sln +++ b/ColorzCore/ColorzCore.sln @@ -1,9 +1,11 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26430.6 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34714.143 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColorzCore", "ColorzCore.csproj", "{B98F7CCF-9CAA-406E-88D7-2040FA99F631}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ColorzCore", "ColorzCore.csproj", "{B98F7CCF-9CAA-406E-88D7-2040FA99F631}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ColorzCore.Framework", "ColorzCore.Framework.csproj", "{33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -18,6 +20,12 @@ Global {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Debug|Any CPU.Build.0 = Debug|Any CPU {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Release|Any CPU.ActiveCfg = Release|Any CPU {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Release|Any CPU.Build.0 = Release|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.32bit|Any CPU.ActiveCfg = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.32bit|Any CPU.Build.0 = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/ColorzCore/DataTypes/CaseInsensitiveString.cs b/ColorzCore/DataTypes/CaseInsensitiveString.cs deleted file mode 100644 index 8fd8ade..0000000 --- a/ColorzCore/DataTypes/CaseInsensitiveString.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.DataTypes -{ - class CaseInsensitiveString - { - private string data; - public string String { - get { return data; } - set - { - data = value.ToUpper(); - } - } - public CaseInsensitiveString(string input) - { - String = input; - } - public override int GetHashCode() - { - return String.GetHashCode(); - } - public override bool Equals(object obj) - { - return String == obj.ToString(); - } - public override string ToString() - { - return String; - } - } -} diff --git a/ColorzCore/DataTypes/Either.cs b/ColorzCore/DataTypes/Either.cs index 5d4896f..77c2476 100644 --- a/ColorzCore/DataTypes/Either.cs +++ b/ColorzCore/DataTypes/Either.cs @@ -14,9 +14,9 @@ public interface Either bool IsRight { get; } Left GetLeft { get; } Right GetRight { get; } - void Case(TAction LAction, TAction RAction); + void Case(Action LAction, Action RAction); } - public class Left : Either + public class Left : Either { public Left(L val) { @@ -27,7 +27,7 @@ public Left(L val) public bool IsRight { get { return false; } } public L GetLeft { get; } public R GetRight { get { throw new WrongEitherException(); } } - public void Case(TAction LAction, TAction RAction) { LAction(GetLeft); } + public void Case(Action LAction, Action RAction) { LAction(GetLeft); } } public class Right : Either { @@ -40,7 +40,7 @@ public Right(R val) public bool IsRight { get { return true; } } public L GetLeft { get { throw new WrongEitherException(); } } public R GetRight { get; } - public void Case(TAction LAction, TAction RAction) { RAction(GetRight); } + public void Case(Action LAction, Action RAction) { RAction(GetRight); } } class WrongEitherException : Exception { } } diff --git a/ColorzCore/DataTypes/Extension.cs b/ColorzCore/DataTypes/Extension.cs index abb0c0b..41d6848 100644 --- a/ColorzCore/DataTypes/Extension.cs +++ b/ColorzCore/DataTypes/Extension.cs @@ -50,6 +50,7 @@ public static int ToInt(this string numString) } } public static Dictionary> AddTo(this Dictionary> self, K key, V val) + where K: notnull { if (self.ContainsKey(key)) self[key].Add(val); @@ -58,8 +59,8 @@ public static Dictionary> AddTo(this Dictionary> se self[key] = new List { val }; } return self; - } - + } + public static void SetBits(this byte[] array, int bitOffset, int bitSize, int value) { array.SetBits(0, bitOffset, bitSize, value); @@ -67,10 +68,10 @@ public static void SetBits(this byte[] array, int bitOffset, int bitSize, int va public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, int value) { - if (bitOffset >= 8) + if (bitOffset >= 8) { array.SetBits(byteOffset + bitOffset / 8, bitOffset % 8, bitSize, value); - return; + return; } long bytes = 0; @@ -95,23 +96,23 @@ public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int array[byteOffset + i] = (byte)(bytes >> i * 8); } - public static void SetBits(this byte[] array, int bitOffset, int bitSize, byte[] data) - { - array.SetBits(0, bitOffset, bitSize, data); + public static void SetBits(this byte[] array, int bitOffset, int bitSize, byte[] data) + { + array.SetBits(0, bitOffset, bitSize, data); } - public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, byte[] data) - { - if (bitOffset >= 8) + public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, byte[] data) + { + if (bitOffset >= 8) { array.SetBits(byteOffset + bitOffset / 8, bitOffset % 8, bitSize, data); - return; + return; } int byteSize = (bitOffset + bitSize + 7) / 8; - if (bitOffset == 0) - { + if (bitOffset == 0) + { for (int i = 0; i < byteSize - 1; ++i) array[byteOffset + i] = data[i]; @@ -119,24 +120,24 @@ public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int uint endMask = (uint)((1 << shift) - 1); array[byteOffset + byteSize - 1] &= (byte)~endMask; - array[byteOffset + byteSize - 1] |= (byte)(data[byteSize - 1] & endMask); + array[byteOffset + byteSize - 1] |= (byte)(data[byteSize - 1] & endMask); } else - { - for (int i = 0; i < byteSize; ++i) - { - int mask = ((1 << Math.Min(bitSize - i * 8, 8)) - 1) << bitOffset; - - byte loMask = (byte)mask; - byte hiMask = (byte)(mask >> 8); - - array[byteOffset + i] &= (byte)~loMask; - array[byteOffset + i] |= (byte)((data[i] << bitOffset) & loMask); - - array[byteOffset + i + 1] &= (byte)~hiMask; - array[byteOffset + i + 1] |= (byte)((data[i] >> (8 - bitOffset)) & hiMask); + { + for (int i = 0; i < byteSize; ++i) + { + int mask = ((1 << Math.Min(bitSize - i * 8, 8)) - 1) << bitOffset; + + byte loMask = (byte)mask; + byte hiMask = (byte)(mask >> 8); + + array[byteOffset + i] &= (byte)~loMask; + array[byteOffset + i] |= (byte)((data[i] << bitOffset) & loMask); + + array[byteOffset + i + 1] &= (byte)~hiMask; + array[byteOffset + i + 1] |= (byte)((data[i] >> (8 - bitOffset)) & hiMask); } - } + } } } } diff --git a/ColorzCore/DataTypes/ImmutableStack.cs b/ColorzCore/DataTypes/ImmutableStack.cs index 51e5aef..ea4a2bb 100644 --- a/ColorzCore/DataTypes/ImmutableStack.cs +++ b/ColorzCore/DataTypes/ImmutableStack.cs @@ -9,42 +9,32 @@ namespace ColorzCore.DataTypes { public class ImmutableStack : IEnumerable { - private Maybe>> member; - int? count; + private readonly Tuple>? member; + private int? count; public ImmutableStack(T elem, ImmutableStack tail) - { member = new Just>>(new Tuple>(elem, tail)); } - private ImmutableStack() { - member = new Nothing>>(); - count = null; + member = new Tuple>(elem, tail); } - private static ImmutableStack emptyList = new ImmutableStack(); - public static ImmutableStack Nil { get { return emptyList;} } - - public bool IsEmpty { get { return member.IsNothing; } } - public T Head { get { return member.FromJust.Item1; } } - public ImmutableStack Tail { get { return member.FromJust.Item2; } } - public int Count { get - { - if (count.HasValue) - return count.Value; - else - return (count = Tail.Count + 1).Value; - } } - /* - public bool Contains(T toLookFor) + private ImmutableStack() { - bool acc = false; - for(ImmutableStack temp = this; !acc && !temp.IsEmpty; temp = temp.Tail) acc |= temp.Head.Equals(toLookFor); - return acc; + member = null; + count = 0; } - */ + + public static ImmutableStack Nil { get; } = new ImmutableStack(); + + public bool IsEmpty => member == null; + public T Head => member!.Item1; + public ImmutableStack Tail => member!.Item2; + public int Count => count ?? (count = Tail.Count + 1).Value; + public IEnumerator GetEnumerator() { ImmutableStack temp = this; - while(!temp.IsEmpty) + + while (!temp.IsEmpty) { yield return temp.Head; temp = temp.Tail; @@ -54,6 +44,7 @@ public IEnumerator GetEnumerator() IEnumerator IEnumerable.GetEnumerator() { ImmutableStack temp = this; + while (!temp.IsEmpty) { yield return temp.Head; @@ -63,7 +54,7 @@ IEnumerator IEnumerable.GetEnumerator() public static ImmutableStack FromEnumerable(IEnumerable content) { - return content.Reverse().Aggregate(Nil, (ImmutableStack acc, T elem) => new ImmutableStack(elem, acc)); + return content.Reverse().Aggregate(Nil, (acc, elem) => new ImmutableStack(elem, acc)); } } } diff --git a/ColorzCore/DataTypes/Location.cs b/ColorzCore/DataTypes/Location.cs index 7699204..4ea0e52 100644 --- a/ColorzCore/DataTypes/Location.cs +++ b/ColorzCore/DataTypes/Location.cs @@ -9,13 +9,20 @@ namespace ColorzCore.DataTypes public struct Location { public string file; - public int lineNum, colNum; + public int line, column; + public MacroLocation? macroLocation; - public Location(string fileName, int lineNum, int colNum) : this() + public Location(string fileName, int lineNum, int colNum, MacroLocation? macro = null) : this() { file = fileName; - this.lineNum = lineNum; - this.colNum = colNum; + line = lineNum; + column = colNum; + macroLocation = macro; } + + public readonly Location OffsetBy(int columns) => new Location(file, line, column + columns, macroLocation); + public readonly Location MacroClone(MacroLocation macro) => new Location(file, line, column, macro); + + public override readonly string ToString() => $"{file}:{line}:{column}"; } } diff --git a/ColorzCore/DataTypes/MacroLocation.cs b/ColorzCore/DataTypes/MacroLocation.cs new file mode 100644 index 0000000..fd50152 --- /dev/null +++ b/ColorzCore/DataTypes/MacroLocation.cs @@ -0,0 +1,14 @@ +namespace ColorzCore.DataTypes +{ + public class MacroLocation + { + public string MacroName { get; } + public Location Location { get; } + + public MacroLocation(string macroName, Location location) + { + MacroName = macroName; + Location = location; + } + } +} diff --git a/ColorzCore/DataTypes/Maybe.cs b/ColorzCore/DataTypes/Maybe.cs deleted file mode 100644 index 130583c..0000000 --- a/ColorzCore/DataTypes/Maybe.cs +++ /dev/null @@ -1,79 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.DataTypes -{ - public delegate R UnaryFunction(T val); - public delegate R RConst(); - public delegate void TAction(T val); - public delegate void NullaryAction(); - public delegate Maybe MaybeAction(T val); - -#pragma warning disable IDE1006 // Naming Styles - public interface Maybe -#pragma warning restore IDE1006 // Naming Styles - { - bool IsNothing { get; } - T FromJust { get; } - Maybe Fmap(UnaryFunction f); - Maybe Bind(MaybeAction f); - R IfJust(UnaryFunction just, RConst nothing); - void IfJust(TAction just, NullaryAction nothing = null); - } - public class Just : Maybe - { - public bool IsNothing { get { return false; } } - public T FromJust { get; } - - public Just(T val) - { - FromJust = val; - } - - public Maybe Fmap(UnaryFunction f) - { - return new Just(f(FromJust)); - } - public Maybe Bind(MaybeAction f) - { - return f(FromJust); - } - public R IfJust(UnaryFunction just, RConst nothing) - { - return just(FromJust); - } - public void IfJust(TAction just, NullaryAction nothing) - { - just(FromJust); - } - } - public class Nothing : Maybe - { - public bool IsNothing { get { return true; } } - public T FromJust { get { throw new MaybeException(); } } - - public Nothing() { } - - public Maybe Fmap(UnaryFunction f) - { - return new Nothing(); - } - public Maybe Bind(MaybeAction f) - { - return new Nothing(); - } - public R IfJust(UnaryFunction just, RConst nothing) - { - return nothing(); - } - public void IfJust(TAction just, NullaryAction nothing) - { - nothing?.Invoke(); - } - } - - public class MaybeException : Exception { } -} diff --git a/ColorzCore/DataTypes/MergeableGenerator.cs b/ColorzCore/DataTypes/MergeableGenerator.cs index a483226..ae7fe97 100644 --- a/ColorzCore/DataTypes/MergeableGenerator.cs +++ b/ColorzCore/DataTypes/MergeableGenerator.cs @@ -7,22 +7,24 @@ namespace ColorzCore.DataTypes { - public class MergeableGenerator: IEnumerable + public class MergeableGenerator : IEnumerable { - private Stack> myEnums; + private readonly Stack> myEnums; public bool EOS { get; private set; } + public MergeableGenerator(IEnumerable baseEnum) { myEnums = new Stack>(); myEnums.Push(baseEnum.GetEnumerator()); } - public T Current { get { return myEnums.Peek().Current; } } + public T Current => myEnums.Peek().Current; + public bool MoveNext() { - if(!myEnums.Peek().MoveNext()) + if (!myEnums.Peek().MoveNext()) { - if(myEnums.Count > 1) + if (myEnums.Count > 1) { myEnums.Pop(); return true; @@ -38,43 +40,55 @@ public bool MoveNext() return true; } } + public void PrependEnumerator(IEnumerator nextEnum) { if (EOS) + { myEnums.Pop(); + } + myEnums.Push(nextEnum); Prime(); } + public void PutBack(T elem) { - this.PrependEnumerator(new List { elem }.GetEnumerator()); + PrependEnumerator(new List { elem }.GetEnumerator()); } + private bool Prime() { if (!myEnums.Peek().MoveNext()) { if (myEnums.Count == 1) EOS = true; - else + else myEnums.Pop(); } else + { EOS = false; + } + return !EOS; } + IEnumerator IEnumerable.GetEnumerator() { - while(!EOS) { - yield return this.Current; - this.MoveNext(); + while (!EOS) + { + yield return Current; + MoveNext(); } } + public IEnumerator GetEnumerator() { while (!EOS) { - yield return this.Current; - this.MoveNext(); + yield return Current; + MoveNext(); } } } diff --git a/ColorzCore/DataTypes/NullableExtensions.cs b/ColorzCore/DataTypes/NullableExtensions.cs new file mode 100644 index 0000000..1a0fef4 --- /dev/null +++ b/ColorzCore/DataTypes/NullableExtensions.cs @@ -0,0 +1,69 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ColorzCore.DataTypes +{ + public static class NullableExtensions + { + public static R? Fmap(this T? self, Func f) + where T : class + { + return self != null ? f(self) : default; + } + + public static R IfJust(this T? self, Func just, Func nothing) + where T : class + { + if (self != null) + { + return just(self); + } + else + { + return nothing(); + } + } + + public static R IfJust(this T? self, Func just, Func nothing) + where T : struct + { + if (self.HasValue) + { + return just(self.Value); + } + else + { + return nothing(); + } + } + + public static void IfJust(this T? self, Action just, Action? nothing = null) + where T : class + { + if (self != null) + { + just(self); + } + else + { + nothing?.Invoke(); + } + } + + public static void IfJust(this T? self, Action just, Action? nothing = null) + where T : struct + { + if (self.HasValue) + { + just(self.Value); + } + else + { + nothing?.Invoke(); + } + } + } +} diff --git a/ColorzCore/EA Standard library/Convo Helpers.txt b/ColorzCore/EA Standard library/Convo Helpers.txt deleted file mode 100644 index fa836a1..0000000 --- a/ColorzCore/EA Standard library/Convo Helpers.txt +++ /dev/null @@ -1,36 +0,0 @@ - -//Show text with background and return to map -#ifdef _FE6_ -#define Text(text) "TEX1 text; REMA" -#define Text(background,text) "FADI 0x10; HIDEMAP; BACG background; FADU 0x10; SHOWMAP; TEX1 text; REMA" -#endif - -#ifdef _FE7_ -#define Text(text) "TEX1 text; REMA" -#define Text(background,text) "FADI 0x10; HIDEMAP; BACG background; FADU 0x10; SHOWMAP; TEX1 text; REMA" -#define ClearBrownBox "_ASM0x42 0x83181" -#endif - -#ifdef _FE8_ -#define Text(text) "TEXTSTART; TEXTSHOW text; TEXTEND; REMA" -#define Text(background,text) "_SETVAL 2 background; _SETVAL 3 text; CALL $9EE310" -#define SetBackground(background) "SVAL 2 background; CALL $9EE2E8" //EVBIT 0x8 = fade in? -#define ClearBackground "CALL 0x9EE2C4" -#define ClearBackgroundSmooth "FADI 0x10; ClearBackground" -//#define SetBackground(background) "SETVAL 0x2 background; EVBIT_CHECK 0x8; REMOVEPORTRAITS; BACG 0xFFFF 0x0 0x0" -#define CenterTutorialTextBox "_SETVAL 0xB 0xFFFFFFFF" -#define FlashWhite "FAWI 0x20; STAL 0x10; FAWU 0x20" -#define FlashBlack "FADI 0x20; STAL 0x10; FADU 0x20" -#define ShowCG(BGIndex) "SETVAL 0x2 BGIndex; REMOVEPORTRAITS; BACG 0xFFFF;" -#endif - - -//Smooth changing to CG -#ifdef _FE7_ -#define ChangeToCutScene(cutscene) "FADICG 0x10; HIDEMAP; SHCG cutscene; FADUCG 0x10; SHOWMAP" -#endif - -//Shows text on a scroll -#ifdef _FE7_ -#define ScrollText(textID) "TEX6 7 [0,0] textID; _ASM0x42 $83181; _0x89" -#endif diff --git a/ColorzCore/EA Standard library/FE8 Definitions.txt b/ColorzCore/EA Standard library/FE8 Definitions.txt deleted file mode 100644 index 01faeac..0000000 --- a/ColorzCore/EA Standard library/FE8 Definitions.txt +++ /dev/null @@ -1,476 +0,0 @@ -//Characters -#define Eirika 0x01 -#define Seth 0x02 -#define Gilliam 0x03 -#define Franz 0x04 -#define Moulder 0x05 -#define Vanessa 0x06 -#define Ross 0x07 -#define Neimi 0x08 -#define Colm 0x09 -#define Garcia 0x0A -#define Innes 0x0B -#define Lute 0x0C -#define Natasha 0x0D -#define Cormag 0x0E -#define Ephraim 0x0F -#define Forde 0x10 -#define Kyle 0x11 -#define Amelia 0x12 -#define Artur 0x13 -#define Gerik 0x14 -#define Tethys 0x15 -#define Marisa 0x16 -#define Saleh 0x17 -#define Ewan 0x18 -#define LArachel 0x19 -#define Dozla 0x1A -#define Rennac 0x1C -#define Duessel 0x1D -#define Myrrh 0x1E -#define Knoll 0x1F -#define Joshua 0x20 -#define Syrene 0x21 -#define Tana 0x22 -#define LyonCC 0x23 -#define OrsonCC 0x24 -#define GlenCC 0x25 -#define SelenaCC 0x26 -#define ValterCC 0x27 -#define RievCC 0x28 -#define CaellachCC 0x29 -#define FadoCC 0x2A -#define IsmaireCC 0x2B -#define HaydenCC 0x2C -#define LyonSummon 0x3B -#define KnollSummon 0x3E -#define EvanSummon 0x3F -#define Lyon_Ch17 0x40 -#define Morva_Ch20 0x41 -#define Orson_Ch5x 0x42 -#define Valter_Ch15_ 0x43 -#define Valter_Ch15 0x43 -#define Selena_Ch10B_and_13B 0x44 -#define Valter_Prologue 0x45 -#define Breguet 0x46 -#define Bone 0x47 -#define Bazba 0x48 -#define Entombed_boss 0x49 -#define Saar 0x4A -#define Novala 0x4B -#define Murray 0x4C -#define Tirado 0x4D -#define Binks 0x4E -#define Pablo 0x4F -#define Macdaire 0x50 -#define Maelduin_boss 0x50 -#define Aias 0x51 -#define Carlyle 0x52 -#define Caellach 0x53 -#define Pablo_2 0x54 -#define Gorgon_boss 0x56 -#define Riev 0x57 -#define Gheb 0x5A -#define Beran 0x5B -#define Cyclops_boss 0x5C -#define Wight_boss 0x5D -#define Deathgoyle_boss 0x5E -#define Bandit 0x66 -#define ONeill 0x68 -#define Glen 0x69 -#define Zonta 0x6A -#define Vigarde 0x6B -#define Lyon_Final 0x6C -#define Orson_Boss 0x6D -#define Fomortiis 0xBE -#define Fado 0xC5 -#define Hayden 0xC7 -#define Mansel 0xC8 -#define Klimt 0xC9 -#define Dara 0xCA -#define Ismaire 0xCB -#define PegasusMessenger 0xCC -#define RiverFolkChild_F 0xF4 -#define RiverFolk_F 0xF5 -#define RiverFolk 0xF6 -#define RiverFolk_F_2 0xF7 -#define RiverFolkChild 0xF8 -#define RenaisCivilianChild_F 0xF9 -#define RenaisCivilian 0xFA -#define RenaisCivilian_F 0xFB -#define OldCivilian 0xFC -#define Wall 0xFE -#define Snag 0xFF - -//Classes -#define EphraimLord 0x01 -#define EirikaLord 0x02 -#define EphraimMasterLord 0x03 -#define EirikaMasterLord 0x04 -#define Cavalier 0x05 -#define Cavalier_F 0x06 -#define Paladin 0x07 -#define Paladin_F 0x08 -#define Knight 0x09 -#define Knight_F 0x0A -#define General 0x0B -#define General_F 0x0C -#define Thief 0x0D -#define Manakete 0x0E -#define Mercenary 0x0F -#define Mercenary_F 0x10 -#define Hero 0x11 -#define Hero_F 0x12 -#define Myrmidon 0x13 -#define Myrmidon_F 0x14 -#define Swordmaster 0x15 -#define Swordmaster_F 0x16 -#define Assassin 0x17 -#define Assassin_F 0x18 -#define Archer 0x19 -#define Archer_F 0x1A -#define Sniper 0x1B -#define Sniper_F 0x1C -#define Ranger 0x1D -#define Ranger_F 0x1E -#define WyvernRider 0x1F -#define WyvernRider_F 0x20 -#define WyvernLord 0x21 -#define WyvernLord_F 0x22 -#define WyvernKnight 0x23 -#define WyvernKnight_F 0x24 -#define Mage 0x25 -#define Mage_F 0x26 -#define Sage 0x27 -#define Sage_F 0x28 -#define MageKnight 0x29 -#define MageKnight_F 0x2A -#define Bishop 0x2B -#define Bishop_F 0x2C -#define Shaman 0x2D -#define Shaman_F 0x2E -#define Druid 0x2F -#define Druid_F 0x30 -#define Summoner 0x31 -#define Summoner_F 0x32 -#define Rogue 0x33 -#define GorgonEgg 0x34 -#define GreatKnight 0x35 -#define GreatKnight_F 0x36 -#define Recruit_2 0x37 -#define Journeyman_3 0x38 -#define Pupil_3 0x39 -#define Recruit_3 0x3A -#define Manakete_2 0x3B -#define Manakete_2_F 0x3C -#define Journeyman_1 0x3D -#define Pupil_1 0x3E -#define Fighter 0x3F -#define Warrior 0x40 -#define Brigand 0x41 -#define Pirate 0x42 -#define Berserker 0x43 -#define Monk 0x44 -#define Priest 0x45 -#define Bard 0x46 -#define Recruit_1 0x47 -#define PegasusKnight 0x48 -#define FalcoKnight 0x49 -#define Cleric 0x4A -#define Troubadour 0x4B -#define Valkyrie 0x4C -#define Dancer 0x4D -#define Soldier 0x4E -#define Necromancer 0x4F -#define Fleet 0x50 -#define GhostFighter 0x51 -#define Revenant 0x52 -#define Entombed 0x53 -#define Bonewalker 0x54 -#define Bonewalker_Bow 0x55 -#define Wight 0x56 -#define Wight_Bow 0x57 -#define Bael 0x58 -#define ElderBael 0x59 -#define Cyclops 0x5A -#define Mauthedoog 0x5B -#define Gwyllgi 0x5C -#define Tarvos 0x5D -#define Maelduin 0x5E -#define Mogall 0x5F -#define ArchMogall 0x60 -#define Gorgon 0x61 -#define GorgonEgg_2 0x62 -#define Gargoyle 0x63 -#define Deathgoyle 0x64 -#define DracoZombie 0x65 -#define DemonKing 0x66 -#define ArcheronBallista 0x67 -#define ArcheronIronBallista 0x68 -#define ArcheronKillerBallista 0x69 -#define EmptyBallista 0x6A -#define EmptyIronBallista 0x6B -#define EmptyKillerBallista 0x6C -#define Civilian 0x6D -#define Civilian_F 0x6E -#define Civilian_2 0x6F -#define Civilian_F_2 0x70 -#define Civilian_3 0x71 -#define Civilian_F_3 0x72 -#define Peer 0x73 -#define Queen 0x74 -#define Prince 0x75 -#define Queen_2 0x76 -#define FallenPrince 0x78 -#define Tent 0x79 -#define Pontifex 0x7A -#define FallenPeer 0x7B -#define Cyclops_2 0x7C -#define ElderBael_2 0x7D -#define Journeyman_2 0x7E -#define Pupil_2 0x7F - -//Items -#define IronSword 0x01 -#define SlimSword 0x02 -#define SteelSword 0x03 -#define SilverSword 0x04 -#define IronBlade 0x05 -#define SteelBlade 0x06 -#define SilverBlade 0x07 -#define PoisonSword 0x08 -#define Rapier 0x09 -#define ManiKatti 0x0A -#define BraveSword 0x0B -#define Shamshir 0x0C -#define KillingEdge 0x0D -#define Armourslayer 0x0E -#define Armorslayer 0x0E -#define Wyrmslayer 0x0F -#define LightBrand 0x10 -#define Lightbrand 0x10 -#define Runesword 0x11 -#define Lancereaver 0x12 -#define Longsword 0x13 -#define Zanbato 0x13 -#define Zanbatou 0x13 -#define IronLance 0x14 -#define SlimLance 0x15 -#define SteelLance 0x16 -#define SilverLance 0x17 -#define PoisonLance 0x18 -#define ToxicLance 0x18 -#define ToxinLance 0x18 -#define BraveLance 0x19 -#define KillerLance 0x1A -#define Horseslayer 0x1B -#define Ridersbane 0x1B -#define Horsekiller 0x1B -#define Javelin 0x1C -#define Spear 0x1D -#define Axereaver 0x1E -#define IronAxe 0x1F -#define SteelAxe 0x20 -#define SilverAxe 0x21 -#define PoisonAxe 0x22 -#define BraveAxe 0x23 -#define KillerAxe 0x24 -#define Halberd 0x25 -#define Hammer 0x26 -#define DevilAxe 0x27 -#define HandAxe 0x28 -#define Tomahawk 0x29 -#define Swordreaver 0x2A -#define Swordslayer 0x2B -#define Hatchet 0x2C -#define IronBow 0x2D -#define SteelBow 0x2E -#define SilverBow 0x2F -#define PoisonBow 0x30 -#define KillerBow 0x31 -#define BraveBow 0x32 -#define ShortBow 0x33 -#define Longbow 0x34 -#define Ballista 0x35 -#define IronBallista 0x36 -#define KillerBallista 0x37 -#define Fire 0x38 -#define Thunder 0x39 -#define Elfire 0x3A -#define Bolting 0x3B -#define Fimbulvetr 0x3C -#define Forblaze 0x3D -#define Excalibur 0x3E -#define Lightning 0x3F -#define Shine 0x40 -#define Divine 0x41 -#define Purge 0x42 -#define Aura 0x43 -#define Luce 0x44 -#define Flux 0x45 -#define Luna 0x46 -#define Nosferatu 0x47 -#define Eclipse 0x48 -#define Fenrir 0x49 -#define Gleipnir 0x4A -#define Heal 0x4B -#define Mend 0x4C -#define Recover 0x4D -#define Physic 0x4E -#define Fortify 0x4F -#define Restore 0x50 -#define Silence 0x51 -#define Sleep 0x52 -#define Berserk 0x53 -#define Warp 0x54 -#define Rescue 0x55 -#define TorchStaff 0x56 -#define Hammerne 0x57 -#define Unlock 0x58 -#define Barrier 0x59 -#define DragonAxe 0x5A -#define AngelicRobe 0x5B -#define EnergyRing 0x5C -#define SecretBook 0x5D -#define Speedwings 0x5E -#define GoddessIcon 0x5F -#define Dragonshield 0x60 -#define Dracoshield 0x60 -#define Talisman 0x61 -#define Boots 0x62 -#define BodyRing 0x63 -#define HeroCrest 0x64 -#define HerosCrest 0x64 -#define KnightCrest 0x65 -#define KnightsCrest 0x65 -#define OrionBolt 0x66 -#define OrionsBolt 0x66 -#define ElysianWhip 0x67 -#define GuidingRing 0x68 -#define ChestKey 0x69 -#define DoorKey 0x6A -#define Lockpick 0x6B -#define Vulnerary 0x6C -#define Elixir 0x6D -#define PureWater 0x6E -#define Antidote 0x6F -#define Antitoxin 0x6F -#define Torch 0x70 -#define TorchItem 0x70 -#define DelphiShield 0x71 -#define MemberCard 0x72 -#define SilverCard 0x73 -#define WhiteGem 0x74 -#define BlueGem 0x75 -#define RedGem 0x76 -#define Gold 0x77 -#define Reginleif 0x78 -#define ChestKey_5 0x79 -#define Mine 0x7A -#define LightRune 0x7B -#define HoplonShield 0x7C -#define FillasMight 0x7D -#define NinissGrace 0x7E -#define ThorsIre 0x7F -#define SetsLitany 0x80 -#define ShadowKiller 0x81 -#define Shadowkiller 0x81 -#define BrightLance 0x82 -#define FiendCleaver 0x83 -#define Fiendcleaver 0x83 -#define BeaconBow 0x84 -#define Sieglind 0x85 -#define Sieglinde 0x85 -#define BattleAxe 0x86 -#define Ivaldi 0x87 -#define MasterProof 0x88 -#define MasterSeal 0x88 -#define MetissTome 0x89 -#define HeavenSeal 0x8A -#define SharpClaw 0x8B -#define Latona 0x8C -#define DragonSpear 0x8D -#define Vidofnir 0x8E -#define Naglfar 0x8F -#define WretchedAir 0x90 -#define Audomra 0x91 -#define Audhulma 0x91 -#define Siegmund 0x92 -#define Garm 0x93 -#define Nidhogg 0x94 -#define HeavySpear 0x95 -#define ShortSpear 0x96 -#define ConquerorsProof 0x97 -#define OceanSeal 0x97 -#define MoonBracelet 0x98 -#define LunarBracelet 0x98 -#define SunBracelet 0x99 -#define SolarBracelet 0x99 -#define _1G 0x9A -#define _5G 0x9B -#define _10G 0x9C -#define _50G 0x9D -#define _100G 0x9E -#define _3000G 0x9F -#define _5000G 0xA0 -#define WindSword 0xA1 -#define Vulnerary_2 0xA2 -#define Greennote 0xA3 -#define Rednote 0xA4 -#define Dance 0xA5 -#define Nightmare 0xA6 -#define NightmareStaff 0xA6 -#define DemonShard 0xA7 -#define DemonStone 0xA7 -#define DemonLight 0xA8 -#define Ravager 0xA9 -#define HolyDragonStone 0xAA -#define DivineDragonStone 0xAA -#define DragonStone 0xAA -#define Dragonstone 0xAA -#define DemonSurge 0xAB -#define Shadowshot 0xAC -#define RottenClaw 0xAD -#define FetidClaw 0xAE -#define PoisonClaw 0xAF -#define LongPoisonClaw 0xB0 -#define LethalTalon 0xB0 -#define FireFang 0xB1 -#define HellFang 0xB2 -#define EvilEye 0xB3 -#define BloodyEye 0xB4 -#define CrimsonEye 0xB4 -#define Stone 0xB5 -#define Aircalibur 0xB6 -#define JunaFruit 0xB7 -#define _150G 0xB8 -#define _200G 0xB9 -#define BlackGem 0xBA -#define GoldGem 0xBB - -#define r0 0x0 -#define r1 0x1 -#define r2 0x2 -#define r3 0x3 -#define r4 0x4 -#define r5 0x5 -#define r6 0x6 -#define r7 0x7 -#define r8 0x8 -#define r9 0x9 -#define rA 0xA -#define r10 0xA -#define rB 0xB -#define r11 0xB -#define rC 0xC -#define r12 0xC -#define rD 0xD -#define r13 0xD -#define QP 0xD //QueuePointer - -#define ActiveUnit (-1) // 0xFFFF -#define UnitAtCoordsSlotB (-2) // 0xFFFE -#define UnitInSlot2 (-3) // 0xFFFD - -#define CurrentUnit ActiveUnit -#define UnitAtCoords UnitAtCoordsSlotB diff --git a/ColorzCore/EADriver.cs b/ColorzCore/EADriver.cs new file mode 100644 index 0000000..b6a7d86 --- /dev/null +++ b/ColorzCore/EADriver.cs @@ -0,0 +1,318 @@ +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; +using ColorzCore.Preprocessor; +using ColorzCore.Preprocessor.Directives; +using ColorzCore.Preprocessor.Macros; +using ColorzCore.Raws; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace ColorzCore +{ + // Class to excapsulate all steps in EA script interpretation (lexing -> parsing -> interpretation -> commit). + class EADriver + { + private Dictionary> allRaws; + private EAParser myParser; + private EAInterpreter myInterpreter; + private string iFile; + private Stream sin; + private Logger Logger { get; } + private IOutput output; + + public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) + { + this.output = output; + + try + { + allRaws = SelectRaws(game, ListAllRaws(rawsFolder, rawsExtension)); + } + catch (RawReader.RawParseException e) + { + Location loc = new Location(e.FileName, e.LineNumber, 1); + + logger.Message(Logger.MessageKind.ERROR, loc, "An error occured while parsing raws"); + logger.Message(Logger.MessageKind.ERROR, loc, e.Message); + + Environment.Exit(-1); // ew? + } + + this.sin = sin; + Logger = logger; + iFile = inFileName; + + myInterpreter = new EAInterpreter(logger); + + ParseConsumerChain parseConsumers = new ParseConsumerChain(); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.SetSymbolMacros)) + { + parseConsumers.Add(new SetSymbolMacroDetector(logger)); + } + + // add the interpreter last + parseConsumers.Add(myInterpreter); + + StringProcessor stringProcessor = new StringProcessor(logger); + + myParser = new EAParser(logger, allRaws, parseConsumers, myInterpreter.BindIdentifier, stringProcessor); + + myParser.Definitions["__COLORZ_CORE__"] = new Definition(); + + myParser.Definitions["__COLORZ_CORE_VERSION__"] = new Definition( + new Token(TokenType.NUMBER, new Location("builtin", 0, 0), "20240504")); + + if (game != null) + { + myParser.Definitions[$"_{game}_"] = new Definition(); + } + + IncludeFileSearcher includeSearcher = new IncludeFileSearcher(); + includeSearcher.IncludeDirectories.Add(AppDomain.CurrentDomain.BaseDirectory); + + foreach (string path in EAOptions.IncludePaths) + { + includeSearcher.IncludeDirectories.Add(path); + } + + myParser.DirectiveHandler.Directives["include"] = new IncludeDirective() { FileSearcher = includeSearcher }; + myParser.DirectiveHandler.Directives["incbin"] = new IncludeBinaryDirective() { FileSearcher = includeSearcher }; + + myParser.DirectiveHandler.Directives["inctbl"] = new IncludeEncodingTableDirective(stringProcessor) + { + FileSearcher = includeSearcher + }; + + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.ReadDataMacros) && output is ROM rom) + { + myParser.Definitions["__has_read_data_macros"] = new Definition(); + + myParser.Macros.BuiltInMacros.Add("ReadByteAt", new ReadDataAt(myParser, rom, 1)); + myParser.Macros.BuiltInMacros.Add("ReadShortAt", new ReadDataAt(myParser, rom, 2)); + myParser.Macros.BuiltInMacros.Add("ReadWordAt", new ReadDataAt(myParser, rom, 4)); + } + else + { + BuiltInMacro unsupportedMacro = new ErrorMacro(myParser, "Macro unsupported in this configuration.", i => i == 1); + + myParser.Macros.BuiltInMacros.Add("ReadByteAt", unsupportedMacro); + myParser.Macros.BuiltInMacros.Add("ReadShortAt", unsupportedMacro); + myParser.Macros.BuiltInMacros.Add("ReadWordAt", unsupportedMacro); + } + + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.IncludeTools)) + { + myParser.Definitions["__has_incext"] = new Definition(); + + IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; + toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); + + foreach (string path in EAOptions.ToolsPaths) + { + includeSearcher.IncludeDirectories.Add(path); + } + + IDirective incextDirective = new IncludeExternalDirective { FileSearcher = toolSearcher }; + IDirective inctextDirective = new IncludeToolEventDirective { FileSearcher = toolSearcher }; + + myParser.DirectiveHandler.Directives["incext"] = incextDirective; + myParser.DirectiveHandler.Directives["inctext"] = inctextDirective; + myParser.DirectiveHandler.Directives["inctevent"] = inctextDirective; + } + + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.AddToPool)) + { + myParser.Definitions["__has_pool"] = new Definition(); + + Pool pool = new Pool(); + + myParser.Macros.BuiltInMacros.Add("AddToPool", new AddToPool(pool)); + myParser.DirectiveHandler.Directives.Add("pool", new PoolDirective(pool)); + } + } + + public bool Interpret() + { + Tokenizer tokenizer = new Tokenizer(); + + ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); + + foreach ((string name, string body) in EAOptions.PreDefintions) + { + Location location = new Location("CMD", 0, 1); + myParser.ParseAll(tokenizer.TokenizePhrase($"#define {name} \"{body}\"", location)); + } + + myParser.ParseAll(tokenizer.Tokenize(sin, iFile)); + + IList lines = myInterpreter.HandleEndOfInput(); + + /* First pass on AST: Identifier resolution. + * + * Suppose we had the code + * + * POIN myLabel + * myLabel: + * + * At parse time, myLabel did not exist for the POIN. + * It is at this point we want to make sure all references to identifiers are valid, before assembling. + */ + List<(Location, Exception)> evaluationErrors = new List<(Location, Exception)>(); + foreach (ILineNode line in lines) + { + try + { + line.EvaluateExpressions(evaluationErrors, EvaluationPhase.Final); + } + catch (MacroInvocationNode.MacroException e) + { + Logger.Error(e.CausedError.MyLocation, "Unexpanded macro."); + } + } + + foreach ((Location location, Exception e) in evaluationErrors) + { + if (e is IdentifierNode.UndefinedIdentifierException uie + && uie.CausedError.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) + { + Logger.Error(location, "Unpooled data (forgot #pool?)"); + } + else + { + Logger.Error(location, e.Message); + } + } + + /* Last step: assembly */ + + ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_DATAWRITE); + + if (!Logger.HasErrored) + { + foreach (ILineNode line in lines) + { + if (Program.Debug) + { + Logger.Message(Logger.MessageKind.DEBUG, line.PrettyPrint(0)); + } + + line.WriteData(output); + } + + output.Commit(); + + Logger.Output.WriteLine("No errors. Please continue being awesome."); + return true; + } + else + { + Logger.Output.WriteLine("Errors occurred; no changes written."); + return false; + } + } + + public bool WriteNocashSymbols(TextWriter output) + { + for (int i = 0; i < myInterpreter.AllScopes.Count; i++) + { + string manglePrefix = string.Empty; + + switch (i) + { + case 0: + output.WriteLine("; global scope"); + break; + default: + output.WriteLine($"; local scope {i}"); + manglePrefix = $"${i}$"; + break; + } + + Closure scope = myInterpreter.AllScopes[i]; + + foreach (KeyValuePair pair in scope.LocalSymbols()) + { + string name = pair.Key; + int address = EAInterpreter.ConvertToAddress(pair.Value); + + output.WriteLine($"{address:X8} {manglePrefix}{name}"); + } + } + + return true; + } + + private static IEnumerable LoadAllRaws(string rawsFolder, string rawsExtension) + { + var directoryInfo = new DirectoryInfo(rawsFolder); + var files = directoryInfo.GetFiles("*" + rawsExtension, SearchOption.AllDirectories); + + foreach (FileInfo fileInfo in files) + { + using var fs = new FileStream(fileInfo.FullName, FileMode.Open); + + foreach (var raw in RawReader.ParseAllRaws(fs)) + yield return raw; + } + } + + private static IList ListAllRaws(string? rawsFolder, string rawsExtension) + { + if (rawsFolder != null) + { + return new List(LoadAllRaws(rawsFolder, rawsExtension)); + } + else + { + return GetFallbackRaws(); + } + } + + private static IList GetFallbackRaws() + { + static List CreateParams(int bitSize, bool isPointer) + { + return new List() { new AtomicParam("Data", 0, bitSize, isPointer) }; + } + + static Raw CreateRaw(string name, int byteSize, int alignment, bool isPointer) + { + return new Raw(name, byteSize * 8, 0, alignment, CreateParams(byteSize * 8, isPointer), true); + } + + return new List() + { + CreateRaw("BYTE", 1, 1, false), + CreateRaw("SHORT", 2, 2, false), + CreateRaw("WORD", 4, 4, false), + CreateRaw("POIN", 4, 4, true), + CreateRaw("SHORT2", 2, 1, false), + CreateRaw("WORD2", 4, 1, false), + CreateRaw("POIN2", 4, 1, true), + }; + } + + private static Dictionary> SelectRaws(string? game, IList allRaws) + { + Dictionary> result = new Dictionary>(); + + foreach (Raw raw in allRaws) + { + if (raw.Game.Count == 0 || (game != null && raw.Game.Contains(game))) + { + result.AddTo(raw.Name, raw); + } + } + + return result; + } + } +} diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs deleted file mode 100644 index c01a7d8..0000000 --- a/ColorzCore/EAInterpreter.cs +++ /dev/null @@ -1,187 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.IO; -using ColorzCore.Lexer; -using ColorzCore.Parser; -using ColorzCore.Parser.AST; -using ColorzCore.Raws; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace ColorzCore -{ - //Class to excapsulate all steps in EA script interpretation. - class EAInterpreter - { - private Dictionary> allRaws; - private EAParser myParser; - private string game, iFile; - private Stream sin; - private Log log; - private IOutput output; - - public EAInterpreter(IOutput output, string game, string rawsFolder, string rawsExtension, Stream sin, string inFileName, Log log) - { - this.game = game; - this.output = output; - - try - { - allRaws = ProcessRaws(game, ListAllRaws(rawsFolder, rawsExtension)); - } - catch (RawReader.RawParseException e) - { - Location loc = new Location - { - file = e.FileName, - lineNum = e.LineNumber, - colNum = 1 - }; - - log.Message(Log.MsgKind.ERROR, loc, "An error occured while parsing raws"); - log.Message(Log.MsgKind.ERROR, loc, e.Message); - - Environment.Exit(-1); // ew? - } - - this.sin = sin; - this.log = log; - iFile = inFileName; - - IncludeFileSearcher includeSearcher = new IncludeFileSearcher(); - includeSearcher.IncludeDirectories.Add(AppDomain.CurrentDomain.BaseDirectory); - - foreach (string path in EAOptions.Instance.includePaths) - includeSearcher.IncludeDirectories.Add(path); - - IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; - toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); - - foreach (string path in EAOptions.Instance.toolsPaths) - includeSearcher.IncludeDirectories.Add(path); - - myParser = new EAParser(allRaws, log, new Preprocessor.DirectiveHandler(includeSearcher, toolSearcher)); - - myParser.Definitions['_' + game + '_'] = new Definition(); - myParser.Definitions["__COLORZ_CORE__"] = new Definition(); - } - - public bool Interpret() - { - Tokenizer t = new Tokenizer(); - - ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - - foreach (Tuple defpair in EAOptions.Instance.defs) - { - myParser.ParseAll(t.TokenizeLine("#define " + defpair.Item1 + " " + defpair.Item2, "cmd", 0)); - } - - IList lines = new List(myParser.ParseAll(t.Tokenize(sin, iFile))); - - /* First pass on AST: Identifier resolution. - * - * Suppose we had the code - * - * POIN myLabel - * myLabel: - * - * At parse time, myLabel did not exist for the POIN. - * It is at this point we want to make sure all references to identifiers are valid, before assembling. - */ - List undefinedIds = new List(); - foreach (ILineNode line in lines) - { - try - { - line.EvaluateExpressions(undefinedIds); - } catch (MacroInvocationNode.MacroException e) - { - myParser.Error(e.CausedError.MyLocation, "Unexpanded macro."); - } - } - - foreach (Token errCause in undefinedIds) - { - if (errCause.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) - { - myParser.Error(errCause.Location, "Unpooled data (forgot #pool?)"); - } - else - { - myParser.Error(errCause.Location, "Undefined identifier: " + errCause.Content); - } - } - - /* Last step: assembly */ - - ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_DATAWRITE); - - if (!log.HasErrored) - { - foreach (ILineNode line in lines) - { - if (Program.Debug) - { - log.Message(Log.MsgKind.DEBUG, line.PrettyPrint(0)); - } - - line.WriteData(output); - } - - output.Commit(); - - log.Output.WriteLine("No errors. Please continue being awesome."); - return true; - } - else - { - log.Output.WriteLine("Errors occurred; no changes written."); - return false; - } - } - - public bool WriteNocashSymbols(TextWriter output) - { - foreach (var label in myParser.GlobalScope.Head.LocalLabels()) - { - // TODO: more elegant offset to address mapping - output.WriteLine("{0:X8} {1}", label.Value + 0x8000000, label.Key); - } - - return true; - } - - private static IEnumerable LoadAllRaws(string rawsFolder, string rawsExtension) - { - var directoryInfo = new DirectoryInfo(rawsFolder); - var files = directoryInfo.GetFiles("*" + rawsExtension, SearchOption.AllDirectories); - - foreach (FileInfo fileInfo in files) - { - using (var fs = new FileStream(fileInfo.FullName, FileMode.Open)) - foreach (var raw in RawReader.ParseAllRaws(fs)) - yield return raw; - } - } - - private static IList ListAllRaws(string rawsFolder, string rawsExtension) - { - return new List(LoadAllRaws(rawsFolder, rawsExtension)); - } - - private static Dictionary> ProcessRaws(string game, IList allRaws) - { - Dictionary> result = new Dictionary>(); - - foreach (Raw r in allRaws) - { - if (r.Game.Count == 0 || r.Game.Contains(game)) - result.AddTo(r.Name, r); - } - - return result; - } - } -} diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index f3d934e..d6d05dd 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -1,37 +1,84 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore { - class EAOptions + public static class EAOptions { - public bool werr; - public bool nowarn, nomess; - public bool buildTimes; + [Flags] + public enum Warnings : long + { + None = 0, + + // warn on non-portable include paths on Windows + NonPortablePath = 1, + + // warn on #define on an existing definition + ReDefine = 2, - public bool noColoredLog; + // warn on write before ORG + // NOTE: currently no way to disable + UninitializedOffset = 4, - public bool nocashSym; + // warn on unintuitive macro expansions (#define A 1 + 2 ... BYTE A * 2 ) + UnintuitiveExpressionMacros = 8, - public List includePaths = new List(); - public List toolsPaths = new List(); - public List> defs = new List>(); - public static EAOptions Instance { get; } = new EAOptions(); + // warn on expansion of unguarded expression within macro + UnguardedExpressionMacros = 16, - public int romOffset; + // warn on macro expanded into "PUSH ; ORG value ; name : ; POP" + // NOTE: currently no way to disable + SetSymbolMacros = 32, - private EAOptions() + // warn on use of legacy features that were superceded (such as the String macro) + LegacyFeatures = 64, + + Extra = UnguardedExpressionMacros | LegacyFeatures, + All = long.MaxValue & ~Extra, + } + + [Flags] + public enum Extensions : long { - werr = false; - nowarn = false; - nomess = false; - noColoredLog = false; - nocashSym = false; - buildTimes = false; - romOffset = 0x8000000; + None = 0, + + // enable ReadByteAt and friends + ReadDataMacros = 1, + + // enable incext and inctext/inctevent + IncludeTools = 2, + + // enable AddToPool and #pool + AddToPool = 4, + + Extra = All, + All = long.MaxValue, } + + public static bool WarningsAreErrors { get; set; } + public static bool QuietWarnings { get; set; } + public static bool QuietMessages { get; set; } + public static bool MonochromeLog { get; set; } + public static bool BenchmarkBuildTimes { get; set; } + public static bool ProduceNocashSym { get; set; } + + // NOTE: currently this is not exposed to users + public static bool TranslateBackslashesInPaths { get; set; } = true; + + public static int BaseAddress { get; set; } = 0x8000000; + public static int MaximumBinarySize { get; set; } = 0x2000000; + + public static List IncludePaths { get; } = new List(); + public static List ToolsPaths { get; } = new List(); + public static List<(string, string)> PreDefintions { get; } = new List<(string, string)>(); + + public static Warnings EnabledWarnings { get; set; } = Warnings.All; + + // NOTE: currently this is not exposed to users + public static Extensions EnabledExtensions { get; set; } = Extensions.All; + + public static bool IsWarningEnabled(Warnings warning) => EnabledWarnings.HasFlag(warning); + public static bool IsExtensionEnabled(Extensions extension) => EnabledExtensions.HasFlag(extension); } } diff --git a/ColorzCore/ExecTimer.cs b/ColorzCore/ExecTimer.cs index 5c603ad..d8ffbb5 100644 --- a/ColorzCore/ExecTimer.cs +++ b/ColorzCore/ExecTimer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; namespace ColorzCore { @@ -14,11 +15,11 @@ public class ExecTimer private List> timingPoints; - private Dictionary times; - private Dictionary counts; + private Dictionary? times; + private Dictionary? counts; + + private TimeSpan totalTime = TimeSpan.Zero; - private TimeSpan totalTime; - private ExecTimer() { timingPoints = new List>(); @@ -37,8 +38,9 @@ public SortedList SortedTimes SortedList sortedTimes = new SortedList(); - foreach (KeyValuePair time in this.times) - sortedTimes.Add(time.Value, time.Key); + if (this.times != null) + foreach (KeyValuePair time in this.times) + sortedTimes.Add(time.Value, time.Key); return sortedTimes; } @@ -51,7 +53,7 @@ public Dictionary Times if (this.times == null) ComputeTimes(); - return this.times; + return this.times!; } } @@ -62,7 +64,7 @@ public Dictionary Counts if (this.counts == null) ComputeTimes(); - return this.counts; + return this.counts!; } } @@ -70,7 +72,7 @@ public TimeSpan TotalTime { get { - if (this.totalTime == null) + if (this.totalTime == TimeSpan.Zero) ComputeTimes(); return this.totalTime; diff --git a/ColorzCore/IO/ASM.cs b/ColorzCore/IO/ASM.cs index 4fb8dfb..66f5cc8 100644 --- a/ColorzCore/IO/ASM.cs +++ b/ColorzCore/IO/ASM.cs @@ -2,8 +2,7 @@ using System.IO; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; +using ColorzCore.Interpreter; namespace ColorzCore.IO { @@ -19,21 +18,21 @@ public ASM(StreamWriter asmStream, StreamWriter ldsStream) private void WriteToASM(int position, byte[] data) { - string sectionName = String.Format(".ea_{0:x}", 0x8000000 + position); - asmStream.WriteLine(".section {0},\"ax\",%progbits", sectionName); - asmStream.WriteLine(".global {0}", sectionName); - asmStream.WriteLine("{0}:", sectionName); + string sectionName = $".ea_{EAInterpreter.ConvertToAddress(position):x}"; + asmStream.WriteLine($".section {sectionName},\"ax\",%progbits"); + asmStream.WriteLine($".global {sectionName}"); + asmStream.WriteLine($"{sectionName}:"); asmStream.Write("\t.byte "); foreach (byte value in data) - asmStream.Write("0x{0:x}, ", value); + asmStream.Write($"0x{value:x}, "); asmStream.WriteLine(); } private void WriteToLDS(int position) { - string sectionName = String.Format(".ea_{0:x}", 0x8000000 + position); - ldsStream.WriteLine(". = 0x{0:x};", 0x8000000 + position); - ldsStream.WriteLine("{0} : {{*.o({0})}}", sectionName); + string sectionName = $".ea_{EAInterpreter.ConvertToAddress(position):x}"; + ldsStream.WriteLine($". = 0x{EAInterpreter.ConvertToAddress(position):x};"); + ldsStream.WriteLine($"{sectionName} : {{*.o({sectionName})}}"); } public void WriteTo(int position, byte[] data) diff --git a/ColorzCore/IO/IOUtility.cs b/ColorzCore/IO/IOUtility.cs index 73c7c78..d7c8fe0 100644 --- a/ColorzCore/IO/IOUtility.cs +++ b/ColorzCore/IO/IOUtility.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -22,17 +23,93 @@ public static string UnescapePath(string param) return sb.Replace("\\ ", " ").Replace("\\\\", "\\").ToString(); } - public static string GetToolFileName(string name) - { - switch (Environment.OSVersion.Platform) + public static string GetToolFileName(string name) + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? $"{name}.exe" : name; + } + + // HACK: this is to make Log not print the entire path every time + public static string GetPortableBasePathForPrefix(string name) + { + string? result = Path.GetDirectoryName(name); + + if (string.IsNullOrEmpty(result)) + { + return ""; + } + else + { + return (result + '/').Replace('\\', '/'); + } + } + + private static readonly char[] pathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; + private static readonly char[] invalidFileCharacters = Path.GetInvalidFileNameChars(); + + /// + /// Convert a user-given path expression to a 'portable' one. That is, it would work on other systems. + /// This is used for warning emission from '#include' and '#incbin' directives that refer to non-portable paths + /// + /// The full real path corresponding to the given expression + /// The user given expression + /// The portable version of the expression, possibly the same as the input expression + public static string GetPortablePathExpression(string fullPath, string pathExpression) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // this method is only meaningful on Windows, which has case-insensitive paths and '\' path separators. + // I believe that on other systems (Linux and friends for sure, probably macOS also?), valid paths are necessarily portable. + return pathExpression; + } + + if (Path.IsPathRooted(pathExpression)) + { + // HACK: rooted paths (like absolute paths) don't really make sense to make portable anyway + // those are most likely generated paths from tools so this doesn't really matter + + return pathExpression; + } + + IList inputComponents = pathExpression.Split(pathSeparators).ToList(); + IList outputComponents = new List(); // will be in reverse + + int upwind = 0; + + foreach (string component in inputComponents.Reverse()) { - case PlatformID.Win32Windows: // Who knows, maybe someone runs EA on win 95 - case PlatformID.Win32NT: - return name + ".exe"; - - default: - return name; - } + if (component.IndexOfAny(invalidFileCharacters) != -1) + { + // fallback just in case + outputComponents.Add(component); + } + if (component == "..") + { + outputComponents.Add(component); + upwind++; + } + else if (component == ".") + { + outputComponents.Add(component); + } + else if (upwind > 0) + { + // this was a "DirName/..", get the real name of this DirName + string? correctComponent = Path.GetFileName(Directory.GetFileSystemEntries(fullPath, component).FirstOrDefault()); + outputComponents.Add(correctComponent ?? component); + + upwind--; + } + else + { + string? reducedPath = Path.GetDirectoryName(fullPath); + fullPath = string.IsNullOrEmpty(reducedPath) ? "." : reducedPath; + + string? correctComponent = Path.GetFileName(Directory.GetFileSystemEntries(fullPath, component).FirstOrDefault()); + outputComponents.Add(correctComponent ?? component); + } + } + + return string.Join("/", outputComponents.Reverse()); } } } diff --git a/ColorzCore/IO/IncludeFileSearcher.cs b/ColorzCore/IO/IncludeFileSearcher.cs index 4fe5f1b..2195ed3 100644 --- a/ColorzCore/IO/IncludeFileSearcher.cs +++ b/ColorzCore/IO/IncludeFileSearcher.cs @@ -10,43 +10,43 @@ public class IncludeFileSearcher public List IncludeDirectories { get; } = new List(); public bool AllowRelativeInclude { get; set; } = true; - public Maybe FindFile(string name) + public string? FindFile(string name) { return FindFile(null, name); } - public Maybe FindFile(string cwd, string name) + public string? FindFile(string? cwd, string name) { // Find the first valid file in the list of possible file paths foreach (string path in EnumeratePossibleAccessPaths(cwd, name)) { if (File.Exists(path)) - return new Just(path); + return path; } - return new Nothing(); + return null; } - public Maybe FindDirectory(string name) + public string? FindDirectory(string name) { return FindDirectory(null, name); } - public Maybe FindDirectory(string cwd, string name) + public string? FindDirectory(string? cwd, string name) { // Find the first valid directory in the list of possible file paths foreach (string path in EnumeratePossibleAccessPaths(cwd, name)) { if (Directory.Exists(path)) - return new Just(path); + return path; } - return new Nothing(); + return null; } - protected IEnumerable EnumeratePossibleAccessPaths(string cwd, string name) + protected IEnumerable EnumeratePossibleAccessPaths(string? cwd, string name) { if (AllowRelativeInclude) { diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Log.cs deleted file mode 100644 index 2cb74af..0000000 --- a/ColorzCore/IO/Log.cs +++ /dev/null @@ -1,83 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using ColorzCore.DataTypes; - -namespace ColorzCore.IO -{ - public class Log - { - public enum MsgKind - { - ERROR, - WARNING, - NOTE, - MESSAGE, - DEBUG, - } - - public bool HasErrored { get; private set; } = false; - public bool WarningsAreErrors { get; set; } = false; - - public bool NoColoredTags { get; set; } = false; - - public List IgnoredKinds { get; } = new List(); - - public TextWriter Output { get; set; } = Console.Error; - - protected struct LogDisplayConfig - { - public string tag; - public ConsoleColor? tagColor; - } - - protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary { - { MsgKind.ERROR, new LogDisplayConfig { tag = "error", tagColor = ConsoleColor.Red } }, - { MsgKind.WARNING, new LogDisplayConfig { tag = "warning", tagColor = ConsoleColor.Magenta } }, - { MsgKind.NOTE, new LogDisplayConfig { tag = "note", tagColor = null } }, - { MsgKind.MESSAGE, new LogDisplayConfig { tag = "message", tagColor = ConsoleColor.Blue } }, - { MsgKind.DEBUG, new LogDisplayConfig { tag = "debug", tagColor = ConsoleColor.Green } } - }; - - public void Message(string message) - { - Message(MsgKind.MESSAGE, null, message); - } - - public void Message(MsgKind kind, string message) - { - Message(kind, null, message); - } - - public void Message(MsgKind kind, Location? source, string message) - { - if (WarningsAreErrors && (kind == MsgKind.WARNING)) - { - kind = MsgKind.ERROR; - } - - HasErrored |= (kind == MsgKind.ERROR); - - if (!IgnoredKinds.Contains(kind)) - { - if (KIND_DISPLAY_DICT.TryGetValue(kind, out LogDisplayConfig config)) - { - if (!NoColoredTags && config.tagColor.HasValue) - Console.ForegroundColor = config.tagColor.Value; - - Output.Write("{0}: ", config.tag); - - if (!NoColoredTags) - Console.ResetColor(); - - if (source.HasValue) - { - Output.Write("{0}:{1}:{2}: ", source.Value.file, source.Value.lineNum, source.Value.colNum); - } - - Output.WriteLine(message); - } - } - } - } -} diff --git a/ColorzCore/IO/Logger.cs b/ColorzCore/IO/Logger.cs new file mode 100644 index 0000000..80d46f9 --- /dev/null +++ b/ColorzCore/IO/Logger.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; + +namespace ColorzCore.IO +{ + public class Logger + { + public enum MessageKind + { + ERROR, + WARNING, + NOTE, + MESSAGE, + DEBUG, + CONTINUE, + } + + public bool HasErrored { get; private set; } = false; + public bool WarningsAreErrors { get; set; } = false; + + public bool NoColoredTags { get; set; } = false; + + public string LocationBasePath { get; set; } = string.Empty; + + public List IgnoredKinds { get; } = new List(); + + public TextWriter Output { get; set; } = Console.Error; + + private MessageKind continueKind = MessageKind.CONTINUE; + private int continueLength = 0; + + protected struct LogDisplayConfig + { + public string tag; + public ConsoleColor? tagColor; + } + + protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary + { + { MessageKind.ERROR, new LogDisplayConfig { tag = "error", tagColor = ConsoleColor.Red } }, + { MessageKind.WARNING, new LogDisplayConfig { tag = "warning", tagColor = ConsoleColor.Magenta } }, + { MessageKind.NOTE, new LogDisplayConfig { tag = "note", tagColor = null } }, + { MessageKind.MESSAGE, new LogDisplayConfig { tag = "message", tagColor = ConsoleColor.Blue } }, + { MessageKind.DEBUG, new LogDisplayConfig { tag = "debug", tagColor = ConsoleColor.Green } } + }; + + public void Message(string message) + { + Message(MessageKind.MESSAGE, null, message); + } + + public void Message(MessageKind kind, string message) + { + Message(kind, null, message); + } + + public void Message(MessageKind kind, Location? source, string message) + { + if (WarningsAreErrors && (kind == MessageKind.WARNING)) + { + kind = MessageKind.ERROR; + } + + HasErrored |= kind == MessageKind.ERROR; + + if (kind == MessageKind.CONTINUE) + { + if (continueKind != MessageKind.CONTINUE && !IgnoredKinds.Contains(continueKind)) + { + Console.Write("".PadLeft(continueLength)); + Console.WriteLine(message); + } + } + else if (!IgnoredKinds.Contains(kind)) + { + if (KIND_DISPLAY_DICT.TryGetValue(kind, out LogDisplayConfig config)) + { + if (!NoColoredTags && config.tagColor.HasValue) + { + Console.ForegroundColor = config.tagColor.Value; + } + + Output.Write($"{config.tag}: "); + continueLength = config.tag.Length + 2; + + if (!NoColoredTags) + Console.ResetColor(); + + if (source.HasValue) + { + string locString = source.Value.ToString(); + + if (locString.StartsWith(LocationBasePath)) + { + locString = locString.Substring(LocationBasePath.Length); + } + + Output.Write($"{locString}: "); + continueLength += locString.Length + 2; + } + + Output.WriteLine(message); + + continueKind = kind; + } + } + } + } +} diff --git a/ColorzCore/IO/LoggerExtensions.cs b/ColorzCore/IO/LoggerExtensions.cs new file mode 100644 index 0000000..8ec85c4 --- /dev/null +++ b/ColorzCore/IO/LoggerExtensions.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; + +namespace ColorzCore.IO +{ + public static class LoggerExtensions + { + // shorthand helpers + + public static void Message(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.MESSAGE, location, message); + public static void Warning(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.WARNING, location, message); + public static void Error(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.ERROR, location, message); + + private static void MessageTrace(this Logger self, Logger.MessageKind kind, Location? location, string message) + { + if (location is Location myLocation && myLocation.macroLocation != null) + { + MacroLocation macroLocation = myLocation.macroLocation; + self.MessageTrace(kind, macroLocation.Location, message); + self.Message(Logger.MessageKind.NOTE, location, $"From inside of macro `{macroLocation.MacroName}`."); + } + else + { + string[] messages = message.Split('\n'); + self.Message(kind, location, messages[0]); + + for (int i = 1; i < messages.Length; i++) + { + self.Message(Logger.MessageKind.CONTINUE, messages[i]); + } + } + } + } +} \ No newline at end of file diff --git a/ColorzCore/IO/ROM.cs b/ColorzCore/IO/ROM.cs index 32b03de..eb81761 100644 --- a/ColorzCore/IO/ROM.cs +++ b/ColorzCore/IO/ROM.cs @@ -2,22 +2,22 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.IO { - class ROM : IOutput + public class ROM : IOutput { - private BufferedStream myStream; - private byte[] myData; + private readonly BufferedStream myStream; + private readonly byte[] myData; private int size; - public ROM(Stream myROM) + public byte this[int offset] => myData[offset]; + + public ROM(Stream myROM, int maximumSize) { myStream = new BufferedStream(myROM); - myData = new byte[0x2000000]; - size = myStream.Read(myData, 0, 0x2000000); + myData = new byte[maximumSize]; + size = myStream.Read(myData, 0, maximumSize); myStream.Position = 0; } diff --git a/ColorzCore/IO/TblEncoding.cs b/ColorzCore/IO/TblEncoding.cs new file mode 100644 index 0000000..68541b8 --- /dev/null +++ b/ColorzCore/IO/TblEncoding.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace ColorzCore.IO +{ + public class TblEncoding + { + // https://datacrystal.romhacking.net/wiki/Text_Table + + private class EncodingTableNode + { + public byte[]? encoding; + public IDictionary nextTable = new Dictionary(); + } + + private readonly IDictionary rootTable; + + private TblEncoding() + { + rootTable = new Dictionary(); + } + + public byte[] ConvertToBytes(string inputString) + { + return ConvertToBytes(inputString, 0, inputString.Length); + } + + public byte[] ConvertToBytes(string inputString, int start, int length) + { + using MemoryStream memoryStream = new MemoryStream(); + using BinaryWriter binaryWriter = new BinaryWriter(memoryStream); + + int offset = start; + int end = start + length; + + while (offset < end) + { + // find longest match by following tree branches and keeping track of last encodable match + + byte[]? currentMatch = null; + int currentMatchLength = 0; + + IDictionary table = rootTable; + + for (int j = 0; offset + j < end; j++) + { + if (!table.TryGetValue(inputString[offset + j], out EncodingTableNode? node)) + { + break; + } + + table = node.nextTable; + + if (node.encoding != null) + { + currentMatch = node.encoding; + currentMatchLength = j + 1; + } + } + + if (currentMatch == null) + { + // TODO: better exception? + throw new Exception($"Could not encode character: '{inputString[offset]}'"); + } + else + { + binaryWriter.Write(currentMatch); + offset += currentMatchLength; + } + } + + return memoryStream.ToArray(); + } + + public void MapSequence(string sequence, byte[] bytes) + { + if (!rootTable.TryGetValue(sequence[0], out EncodingTableNode? mapping)) + { + mapping = rootTable[sequence[0]] = new EncodingTableNode(); + } + + // Find node corresponding to character sequence + for (int i = 1; i < sequence.Length; i++) + { + IDictionary thisTable = mapping.nextTable; + + if (!thisTable.TryGetValue(sequence[i], out mapping)) + { + mapping = thisTable[sequence[i]] = new EncodingTableNode(); + } + } + + mapping.encoding = bytes; + } + + // TODO: this could be elsewhere + private static byte[] HexToBytes(string hex, int offset, int length) + { + byte[] result = new byte[length / 2]; + + for (int i = 0; i < result.Length; i++) + { + // this could be optimized... + result[i] = Convert.ToByte(hex.Substring(offset + i * 2, 2), 16); + } + + return result; + } + + public static TblEncoding FromTextReader(TextReader reader) + { + TblEncoding result = new TblEncoding(); + string? line; + + char[] equalForSplit = new char[] { '=' }; + + while ((line = reader.ReadLine()) != null) + { + string[] parts; + + // this is never null but might be white space + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + line = line.TrimStart(null); + + switch (line[0]) + { + case '*': + result.MapSequence("\n", HexToBytes(line, 1, line.Length - 1)); + break; + + case '/': + // What is this? + // not very well documented + break; + + default: + parts = line.Split(equalForSplit, 2); + result.MapSequence(parts[1], HexToBytes(parts[0], 0, parts[0].Length)); + break; + } + } + + return result; + } + } +} diff --git a/ColorzCore/Interpreter/AtomVisitor.cs b/ColorzCore/Interpreter/AtomVisitor.cs new file mode 100644 index 0000000..023c73c --- /dev/null +++ b/ColorzCore/Interpreter/AtomVisitor.cs @@ -0,0 +1,32 @@ + +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter +{ + public abstract class AtomVisitor + { + protected void Visit(IAtomNode node) + { + switch (node) + { + case OperatorNode operatorNode: + VisitNode(operatorNode); + break; + case UnaryOperatorNode unaryOperatorNode: + VisitNode(unaryOperatorNode); + break; + case IdentifierNode identifierNode: + VisitNode(identifierNode); + break; + case NumberNode numberNode: + VisitNode(numberNode); + break; + } + } + + protected abstract void VisitNode(UnaryOperatorNode node); + protected abstract void VisitNode(IdentifierNode node); + protected abstract void VisitNode(NumberNode node); + protected abstract void VisitNode(OperatorNode node); + } +} diff --git a/ColorzCore/Interpreter/BaseClosure.cs b/ColorzCore/Interpreter/BaseClosure.cs new file mode 100644 index 0000000..5fbe44c --- /dev/null +++ b/ColorzCore/Interpreter/BaseClosure.cs @@ -0,0 +1,14 @@ +namespace ColorzCore.Interpreter +{ + class BaseClosure : Closure + { + public override bool HasLocalSymbol(string label) + { + return label.ToUpperInvariant() switch + { + "CURRENTOFFSET" or "__LINE__" or "__FILE__" => true, + _ => base.HasLocalSymbol(label), + }; + } + } +} diff --git a/ColorzCore/Interpreter/Closure.cs b/ColorzCore/Interpreter/Closure.cs new file mode 100644 index 0000000..c2aa981 --- /dev/null +++ b/ColorzCore/Interpreter/Closure.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter +{ + public class Closure + { + private Dictionary Symbols { get; } + private Dictionary NonComputedSymbols { get; } + + public Closure() + { + Symbols = new Dictionary(); + NonComputedSymbols = new Dictionary(); + } + + public virtual bool HasLocalSymbol(string label) + { + return Symbols.ContainsKey(label) || NonComputedSymbols.ContainsKey(label); + } + + // HACK: having evaluationPhase being passed here is a bit suspect. + public virtual int GetSymbol(string label, EvaluationPhase evaluationPhase) + { + if (Symbols.TryGetValue(label, out int value)) + { + return value; + } + + // Try to evaluate assigned symbol + + IAtomNode node = NonComputedSymbols[label]; + NonComputedSymbols.Remove(label); + + return node.TryEvaluate(e => + { + NonComputedSymbols.Add(label, node); + throw new SymbolComputeException(label, node, e); + }, evaluationPhase)!.Value; + } + + public void AddSymbol(string label, int value) => Symbols[label] = value; + public void AddSymbol(string label, IAtomNode node) => NonComputedSymbols[label] = node.Simplify(e => { }, EvaluationPhase.Early); + + public IEnumerable> LocalSymbols() + { + foreach (KeyValuePair label in Symbols) + { + yield return label; + } + } + + public class SymbolComputeException : Exception + { + public string SymbolName { get; } + public IAtomNode Expression { get; } + + public SymbolComputeException(string name, IAtomNode node, Exception e) + : base($"Couldn't evaluate value of symbol `{name}` ({e.Message})") + { + SymbolName = name; + Expression = node; + } + } + } +} \ No newline at end of file diff --git a/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs new file mode 100644 index 0000000..5d50daa --- /dev/null +++ b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs @@ -0,0 +1,178 @@ +using System; +using System.Collections.Generic; +using System.Text; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter.Diagnostics +{ + public static class DiagnosticsHelpers + { + // Gets string representation of expression with emphasis on locations for which the predicate is true + public static string GetEmphasizedExpression(IAtomNode node, Func emphasisPredicate) + { + return new EmphasisExpressionPrinter(emphasisPredicate).PrintExpression(node); + } + + /* + // Print expression (unused) + public static string PrettyPrintExpression(IAtomNode node) + { + return GetEmphasizedExpression(node, _ => false); + } + */ + + // visits operators that aren't around parenthesises or brackets + public static void VisitUnguardedOperators(IList tokens, Action action) + { + if (tokens.Count > 1) + { + int paren = 0; + int bracket = 0; + + foreach (Token token in tokens) + { + switch (token.Type) + { + case TokenType.OPEN_PAREN: + paren++; + break; + + case TokenType.CLOSE_PAREN: + paren--; + break; + + case TokenType.OPEN_BRACKET: + bracket++; + break; + + case TokenType.CLOSE_BRACKET: + bracket--; + break; + + default: + if (paren == 0 && bracket == 0 && AtomParser.IsInfixOperator(token)) + { + action.Invoke(token); + } + + break; + } + } + } + } + + // helper for DoesOperationSpanMultipleMacrosUnintuitively + private static IAtomNode GetLeftmostInnerNode(IAtomNode node) + { + if (node is OperatorNode operatorNode) + { + return GetLeftmostInnerNode(operatorNode.Left); + } + + return node; + } + + // helper for DoesOperationSpanMultipleMacrosUnintuitively + private static IAtomNode GetRightmostInnerNode(IAtomNode node) + { + if (node is OperatorNode operatorNode) + { + return GetRightmostInnerNode(operatorNode.Right); + } + + if (node is UnaryOperatorNode unaryOperatorNode) + { + return GetRightmostInnerNode(unaryOperatorNode.Inner); + } + + return node; + } + + // returns true if node spans multiple macros unintuitively + public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode operatorNode) + { + /* The condition for this diagnostic are as follows: + * 1. The operator node is from the same macro expansion as the closest node on either side + * 2. The operator node is not from the same macro expansion as the operator token on that same side */ + + MacroLocation? macroLocation = operatorNode.MyLocation.macroLocation; + + IAtomNode left = operatorNode.Left; + IAtomNode right = operatorNode.Right; + + if (left is OperatorNode leftNode) + { + if (macroLocation == GetRightmostInnerNode(left).MyLocation.macroLocation) + { + if (macroLocation != leftNode.OperatorToken.Location.macroLocation) + { + return true; + } + } + } + + if (right is OperatorNode rightNode) + { + if (macroLocation == GetLeftmostInnerNode(right).MyLocation.macroLocation) + { + if (macroLocation != rightNode.OperatorToken.Location.macroLocation) + { + return true; + } + } + } + + return false; + } + + public static string PrettyParamType(ParamType paramType) => paramType switch + { + ParamType.ATOM => "Atom", + ParamType.LIST => "List", + ParamType.STRING => "String", + ParamType.MACRO => "Macro", + _ => "", + }; + + // absolute as in "not relative to the value of a symbol" + // NOTE: currently unused (I considered using this for SetSymbol detection) + public static bool IsAbsoluteAtom(IAtomNode node) => node switch + { + IdentifierNode => false, + + OperatorNode operatorNode => operatorNode.OperatorToken.Type switch + { + // A + B is not absolute if either one is relative, but not both + TokenType.ADD_OP => IsAbsoluteAtom(operatorNode.Left) == IsAbsoluteAtom(operatorNode.Right), + + // A - B is not absolute if A is relative and not B + TokenType.SUB_OP => IsAbsoluteAtom(operatorNode.Left) || !IsAbsoluteAtom(operatorNode.Right), + + // A ?? B is not absolute if A and B aren't absolute + TokenType.UNDEFINED_COALESCE_OP => IsAbsoluteAtom(operatorNode.Left) || IsAbsoluteAtom(operatorNode.Right), + + _ => true, + }, + + _ => true, + }; + + public static bool IsSubtractionOfCurrentOffset(IAtomNode node) + { + if (node is OperatorNode operatorNode && operatorNode.OperatorToken.Type == TokenType.SUB_OP) + { + // the "CURRENTOFFSET" node is a number node whose source token is an identifier + if (operatorNode.Right is NumberNode numberNode) + { + Token token = numberNode.SourceToken; + return token.Type == TokenType.IDENTIFIER && token.Content.ToUpperInvariant() == "CURRENTOFFSET"; + } + } + + return false; + } + } +} diff --git a/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs new file mode 100644 index 0000000..bb4a23d --- /dev/null +++ b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs @@ -0,0 +1,83 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter.Diagnostics +{ + // Helper class for printing expressions (IAtomNode) with some bits emphasized + public class EmphasisExpressionPrinter : AtomVisitor + { + readonly StringBuilder stringBuilder = new StringBuilder(); + readonly StringBuilder underlineBuilder = new StringBuilder(); + + readonly Func emphasisPredicate; + bool wasEmphasized = false; + + public EmphasisExpressionPrinter(Func predicate) + { + emphasisPredicate = predicate; + // targetMacroLocation = macroLocation; + } + + public string PrintExpression(IAtomNode expression) + { + stringBuilder.Clear(); + underlineBuilder.Clear(); + + Visit(expression); + + return $"{stringBuilder}\n{underlineBuilder}"; + } + + private void AppendString(bool strong, string value) + { + // HACK: on Windows, ANSI terminal escape codes don't (always?) work + if (strong && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EAOptions.MonochromeLog) + { + stringBuilder.Append("\x1B[1;37m"); + stringBuilder.Append(value); + stringBuilder.Append("\x1B[0m"); + } + else + { + stringBuilder.Append(value); + } + + underlineBuilder.Append(strong ? '~' : ' ', value.Length); + + wasEmphasized = strong; + } + + protected override void VisitNode(OperatorNode node) + { + AppendString(emphasisPredicate(node.Left.MyLocation), "("); + + Visit(node.Left); + + AppendString(emphasisPredicate(node.OperatorToken.Location), $" {node.OperatorString} "); + + Visit(node.Right); + + AppendString(wasEmphasized, ")"); + } + + protected override void VisitNode(UnaryOperatorNode node) + { + AppendString(emphasisPredicate(node.OperatorToken.Location), node.OperatorString); + + Visit(node.Inner); + } + + protected override void VisitNode(IdentifierNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.IdentifierToken.Content); + } + + protected override void VisitNode(NumberNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.PrettyPrint()); + } + } +} diff --git a/ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs b/ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs new file mode 100644 index 0000000..30bf2e7 --- /dev/null +++ b/ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; +using ColorzCore.Raws; + +namespace ColorzCore.Interpreter.Diagnostics +{ + public class SetSymbolMacroDetector : IParseConsumer + { + public Logger Logger { get; } + + private enum State + { + AwaitingPush, + AwaitingOrgAbsolute, + AwaitingLabel, + AwaitingPop, + AwaitingEndOfMacro, + } + + State suspectState = State.AwaitingPush; + Location? suspectLocation = null; + + public SetSymbolMacroDetector(Logger logger) + { + Logger = logger; + suspectState = State.AwaitingPush; + suspectLocation = null; + } + + private void OnSequenceBroken(Location location) + { + if (suspectState == State.AwaitingEndOfMacro && location.macroLocation != suspectLocation?.macroLocation) + { + Logger.Warning(suspectLocation, + "This looks like the expansion of a \"SetSymbol\" macro.\n" + + "SetSymbol macros are macros defined as `PUSH ; ORG value ; name : ; POP`.\n" + + "Because of changes to the behavior of labels, their use may introduce bugs.\n" + + "Consider using symbol assignment instead by writing `name := value`."); + } + + suspectState = State.AwaitingPush; + suspectLocation = null; + } + + public void OnPushStatement(Location location) + { + OnSequenceBroken(location); + + suspectState = State.AwaitingOrgAbsolute; + suspectLocation = location; + } + + public void OnOrgStatement(Location location, IAtomNode offsetNode) + { + if (suspectState == State.AwaitingOrgAbsolute && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingLabel; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnLabel(Location location, string name) + { + if (suspectState == State.AwaitingLabel && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingPop; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnPopStatement(Location location) + { + if (suspectState == State.AwaitingPop && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingEndOfMacro; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnOpenScope(Location location) => OnSequenceBroken(location); + + public void OnCloseScope(Location location) => OnSequenceBroken(location); + + public void OnRawStatement(Location location, Raw raw, IList parameters) => OnSequenceBroken(location); + + public void OnAssertStatement(Location location, IAtomNode node) => OnSequenceBroken(location); + + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) => OnSequenceBroken(location); + + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) => OnSequenceBroken(location); + + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) => OnSequenceBroken(location); + + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) => OnSequenceBroken(location); + + public void OnData(Location location, byte[] data) => OnSequenceBroken(location); + } +} diff --git a/ColorzCore/Interpreter/EAInterpreter.cs b/ColorzCore/Interpreter/EAInterpreter.cs new file mode 100644 index 0000000..f132421 --- /dev/null +++ b/ColorzCore/Interpreter/EAInterpreter.cs @@ -0,0 +1,496 @@ +using System.Collections.Generic; +using System.Linq; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.Raws; + +namespace ColorzCore.Interpreter +{ + public class EAInterpreter : IParseConsumer + { + public int CurrentOffset => currentOffset; + + public IList AllScopes { get; } + + public ImmutableStack GlobalScope { get; } + public ImmutableStack CurrentScope { get; set; } + + private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized + private readonly IList<(int, int, Location)> protectedRegions; + + private bool diagnosedOverflow; // false until first overflow diagnostic. + private bool offsetInitialized; // false until first ORG, used for diagnostics + private int currentOffset; + + public Logger Logger { get; } + + public EAInterpreter(Logger logger) + { + Closure headScope = new BaseClosure(); + AllScopes = new List() { headScope }; + GlobalScope = new ImmutableStack(headScope, ImmutableStack.Nil); + CurrentScope = GlobalScope; + pastOffsets = new Stack<(int, bool)>(); + protectedRegions = new List<(int, int, Location)>(); + currentOffset = 0; + diagnosedOverflow = false; + offsetInitialized = false; + Logger = logger; + } + + // TODO: these next two functions should probably be moved into their own module + + public static int ConvertToAddress(int value) + { + if (value >= 0 && value < EAOptions.MaximumBinarySize) + { + value += EAOptions.BaseAddress; + } + + return value; + } + + public static int ConvertToOffset(int value) + { + if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) + { + value -= EAOptions.BaseAddress; + } + + return value; + } + + public IAtomNode BindIdentifier(Token identifierToken) + { + return identifierToken.Content.ToUpperInvariant() switch + { + "CURRENTOFFSET" => new NumberNode(identifierToken, EAOptions.BaseAddress + CurrentOffset), + _ => new IdentifierNode(identifierToken, CurrentScope), + }; + } + + // Helper method for statement handlers + private int? EvaluteAtom(IAtomNode node) + { + return node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), EvaluationPhase.Immediate); + } + + private IList LineNodes { get; } = new List(); + + public IList HandleEndOfInput() + { + if (CurrentScope != GlobalScope) + { + Logger.Error(null, "Reached end of input with an open local scope."); + } + + return LineNodes; + } + + #region AST Handlers + + public void OnOpenScope(Location _) + { + Closure newClosure = new Closure(); + + AllScopes.Add(newClosure); + CurrentScope = new ImmutableStack(newClosure, CurrentScope); + } + + public void OnCloseScope(Location location) + { + if (CurrentScope != GlobalScope) + { + CurrentScope = CurrentScope.Tail; + } + else + { + Logger.Error(location, "Attempted to close local scope without being within one."); + } + } + + public void OnRawStatement(Location location, Raw raw, IList parameters) + { + RawNode node = new RawNode(raw, CurrentOffset, parameters); + + if ((CurrentOffset % node.Raw.Alignment) != 0) + { + Logger.Error(location, + $"Bad alignment for raw {node.Raw.Name}: offset ({CurrentOffset:X8}) needs to be {node.Raw.Alignment}-aligned."); + } + else + { + // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + CheckWriteBytes(location, node.Size); + LineNodes.Add(node); + } + } + + public void OnOrgStatement(Location location, IAtomNode offsetNode) + { + if (EvaluteAtom(offsetNode) is int offset) + { + offset = ConvertToOffset(offset); + + if (!TrySetCurrentOffset(offset)) + { + Logger.Error(location, $"Invalid offset: 0x{offset:X}"); + } + else + { + diagnosedOverflow = false; + offsetInitialized = true; + } + } + else + { + // EvaluateAtom already printed an error message + } + } + + public void OnPushStatement(Location _) + { + pastOffsets.Push((CurrentOffset, offsetInitialized)); + } + + public void OnPopStatement(Location location) + { + if (pastOffsets.Count == 0) + { + Logger.Error(location, "POP without matching PUSH."); + } + else + { + (currentOffset, offsetInitialized) = pastOffsets.Pop(); + } + } + + public void OnAssertStatement(Location location, IAtomNode node) + { + // helper for distinguishing boolean expressions and other expressions + // TODO: move elsewhere perhaps + static bool IsBooleanResultHelper(IAtomNode node) + { + return node switch + { + UnaryOperatorNode uon => uon.OperatorToken.Type switch + { + TokenType.LOGNOT_OP => true, + _ => false, + }, + + OperatorNode on => on.OperatorToken.Type switch + { + TokenType.LOGAND_OP => true, + TokenType.LOGOR_OP => true, + TokenType.COMPARE_EQ => true, + TokenType.COMPARE_NE => true, + TokenType.COMPARE_GT => true, + TokenType.COMPARE_GE => true, + TokenType.COMPARE_LE => true, + TokenType.COMPARE_LT => true, + _ => false, + }, + + _ => false, + }; + } + + bool isBoolean = IsBooleanResultHelper(node); + bool isSubtractionOfCurrentOffset = DiagnosticsHelpers.IsSubtractionOfCurrentOffset(node); + + if (EvaluteAtom(node) is int result) + { + if (isBoolean && result == 0) + { + Logger.Error(location, "Assertion failed"); + } + else if (!isBoolean && result < 0) + { + /* users may do something like ASSERT UpperBound - CURRENTOFFSET + * If UpperBound is an offset rather than an address, this will now certainly fail. + * as CURRENTOFFSET was changed to be expanded into an address rather than a ROM offset. + * we do not want this to break too hard so we try to emit a warning rather than an error. */ + + if (isSubtractionOfCurrentOffset && result + EAOptions.BaseAddress >= 0) + { + Logger.Warning(location, + $"Assertion would fail with value {result} if CURRENTOFFSET is treated as an address.\n" + + "ColorzCore was recently changed to work in terms of addresses rather that ROM offsets.\n" + + "Consider changing the left operand to an address, or if you need to keep compatibility,\n" + + "You can also bitwise AND CURRENTOFFSET to keep it within the desired bounds."); + } + else + { + Logger.Error(location, $"Assertion failed with value {result}."); + } + } + } + else + { + Logger.Error(node.MyLocation, "Failed to evaluate ASSERT expression."); + } + } + + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) + { + if (EvaluteAtom(beginAtom) is int beginValue) + { + beginValue = ConvertToOffset(beginValue); + + int length = 4; + + if (endAtom != null) + { + if (EvaluteAtom(endAtom) is int endValue) + { + endValue = ConvertToOffset(endValue); + + length = endValue - beginValue; + + switch (length) + { + case < 0: + Logger.Error(location, $"Invalid PROTECT region: end address ({endValue:X8}) is before start address ({beginValue:X8})."); + return; + + case 0: + // NOTE: does this need to be an error? + Logger.Error(location, $"Empty PROTECT region: end address is equal to start address ({beginValue:X8})."); + return; + } + } + else + { + // EvaluateAtom already printed an error message + return; + } + } + + protectedRegions.Add((beginValue, length, location)); + } + else + { + // EvaluateAtom already printed an error message + } + } + + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) + { + if (EvaluteAtom(alignNode) is int alignValue) + { + if (alignValue > 0) + { + int alignOffset = 0; + + if (offsetNode != null) + { + if (EvaluteAtom(offsetNode) is int rawOffset) + { + if (rawOffset >= 0) + { + alignOffset = ConvertToOffset(rawOffset) % alignValue; + } + else + { + Logger.Error(location, $"ALIGN offset cannot be negative (got {rawOffset})"); + return; + } + } + else + { + // EvaluateAtom already printed an error message + return; + } + } + + if (CurrentOffset % alignValue != alignOffset) + { + int skip = alignValue - (CurrentOffset + alignValue - alignOffset) % alignValue; + + if (!TrySetCurrentOffset(CurrentOffset + skip) && !diagnosedOverflow) + { + Logger.Error(location, $"Trying to jump past end of binary (CURRENTOFFSET would surpass {EAOptions.MaximumBinarySize:X})"); + diagnosedOverflow = true; + } + } + } + else + { + Logger.Error(location, $"Invalid ALIGN value (got {alignValue})."); + } + } + else + { + // EvaluateAtom already printed an error message + } + } + + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) + { + if (EvaluteAtom(amountNode) is int amount) + { + if (amount > 0) + { + int fillValue = 0; + + if (valueNode != null) + { + if (EvaluteAtom(valueNode) is int rawValue) + { + fillValue = rawValue; + } + else + { + // EvaluateAtom already printed an error message + return; + } + } + + var data = new byte[amount]; + + for (int i = 0; i < amount; ++i) + { + data[i] = (byte)fillValue; + } + + var node = new DataNode(CurrentOffset, data); + + CheckWriteBytes(location, amount); + LineNodes.Add(node); + } + else + { + Logger.Error(location, $"Invalid FILL amount (got {amount})."); + } + } + else + { + // EvaluateAtom already printed an error message + } + } + + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) + { + if (atom.TryEvaluate(_ => { }, EvaluationPhase.Early) is int value) + { + TryDefineSymbol(location, name, value); + } + else + { + TryDefineSymbol(location, name, atom); + } + } + + public void OnLabel(Location location, string name) + { + TryDefineSymbol(location, name, ConvertToAddress(CurrentOffset)); + } + + public void OnData(Location location, byte[] data) + { + DataNode node = new DataNode(CurrentOffset, data); + CheckWriteBytes(location, node.Size); + LineNodes.Add(node); + } + + #endregion + + public bool IsValidLabelName(string name) + { + // TODO: this could be where checks for CURRENTOFFSET, __LINE__ and __FILE__ are? + return true; // !IsReservedName(name); + } + + public void TryDefineSymbol(Location location, string name, int value) + { + if (CurrentScope.Head.HasLocalSymbol(name)) + { + Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Logger.Error(location, $"Invalid symbol name {name}."); + } + else + { + CurrentScope.Head.AddSymbol(name, value); + } + } + + public void TryDefineSymbol(Location location, string name, IAtomNode expression) + { + if (CurrentScope.Head.HasLocalSymbol(name)) + { + Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Logger.Error(location, $"Invalid symbol name {name}."); + } + else + { + CurrentScope.Head.AddSymbol(name, expression); + } + } + + // Return value: Location where protection occurred. Nothing if location was not protected. + public Location? IsProtected(int offset, int length) + { + offset = ConvertToOffset(offset); + + foreach ((int protectedOffset, int protectedLength, Location location) in protectedRegions) + { + /* They intersect if the last offset in the given region is after the start of this one + * and the first offset in the given region is before the last of this one. */ + + if (offset + length > protectedOffset && offset < protectedOffset + protectedLength) + { + return location; + } + } + + return null; + } + + public void CheckWriteBytes(Location location, int length) + { + if (!offsetInitialized && EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) + { + Logger.Warning(location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + } + + if (IsProtected(CurrentOffset, length) is Location prot) + { + Logger.Error(location, $"Trying to write data to area protected by {prot}"); + } + + if (!TrySetCurrentOffset(CurrentOffset + length) && !diagnosedOverflow) + { + Logger.Error(location, $"Trying to write past end of binary (CURRENTOFFSET would surpass {EAOptions.MaximumBinarySize:X})"); + diagnosedOverflow = true; + } + } + + private bool TrySetCurrentOffset(int value) + { + if (value < 0 || value > EAOptions.MaximumBinarySize) + { + return false; + } + else + { + currentOffset = value; + diagnosedOverflow = false; + offsetInitialized = true; + return true; + } + } + } +} diff --git a/ColorzCore/Interpreter/EvaluationPhase.cs b/ColorzCore/Interpreter/EvaluationPhase.cs new file mode 100644 index 0000000..3bda589 --- /dev/null +++ b/ColorzCore/Interpreter/EvaluationPhase.cs @@ -0,0 +1,16 @@ +using System; + +namespace ColorzCore.Interpreter +{ + public enum EvaluationPhase + { + // Early-evaluation: we are not done with parsing and obtaining a value now would just be an optimization + Early, + + // Final-evaluation: we are done with parsing and need to freeze this value now + Final, + + // Immediate-evaluation: we are not done with parsing but need this value now + Immediate, + } +} diff --git a/ColorzCore/Interpreter/StringProcessor.cs b/ColorzCore/Interpreter/StringProcessor.cs new file mode 100644 index 0000000..cdaf706 --- /dev/null +++ b/ColorzCore/Interpreter/StringProcessor.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter +{ + // String handling the various processing of strings (encoding, formatting) + public class StringProcessor + { + public IDictionary TableEncodings { get; } = new Dictionary(); + public Logger Logger { get; } + + public StringProcessor(Logger logger) + { + Logger = logger; + } + + public byte[] EncodeString(Location locationForLogger, string inputString, string? encodingName) + { + encodingName ??= "UTF-8"; + + switch (encodingName.ToUpperInvariant()) + { + case "UTF-8": + case "UTF8": + try + { + return Encoding.UTF8.GetBytes(inputString); + } + catch (Exception e) + { + Logger.Error(locationForLogger, + $"Failed to encode string '{inputString}': {e.Message}."); + return Array.Empty(); + } + } + + if (TableEncodings.TryGetValue(encodingName, out TblEncoding? encoding)) + { + try + { + return encoding.ConvertToBytes(inputString); + } + catch (Exception e) + { + Logger.Error(locationForLogger, + $"Failed to encode string '{inputString}': {e.Message}."); + } + } + else + { + Logger.Error(locationForLogger, + $"Unknown encoding: '{encodingName}'."); + } + + return Array.Empty(); + } + + private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); + + string UserFormatStringError(Location loc, string message, string details) + { + Logger.Error(loc, $"An error occurred while expanding format string ({message})."); + return $"<{message}: {details}>"; + } + + public string ExpandUserFormatString(Location location, EAParser parser, string stringValue) + { + return formatItemRegex.Replace(stringValue, match => + { + string expr = match.Groups["expr"].Value!; + string? format = match.Groups["format"].Value; + + return ExpandFormatItem(parser, location.OffsetBy(match.Index), expr, format); + }); + } + + public string ExpandFormatItem(EAParser parser, Location location, string expr, string? format = null) + { + MergeableGenerator tokens = new MergeableGenerator( + Tokenizer.TokenizeLine($"{expr} \n", location)); + + tokens.MoveNext(); + + IParamNode? node = parser.ParseParam(tokens); + + if (node == null || tokens.Current.Type != TokenType.NEWLINE) + { + return UserFormatStringError(location, "bad expression", $"'{expr}'"); + } + + switch (node) + { + case IAtomNode atom: + if (atom.TryEvaluate(e => Logger.Error(atom.MyLocation, e.Message), + EvaluationPhase.Immediate) is int value) + { + try + { + // TODO: do we need to raise an error when result == format? + // this happens (I think) when a custom format specifier with no substitution + return value.ToString(format, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + return UserFormatStringError(node.MyLocation, + "bad format specifier", $"'{format}' ({e.Message})"); + } + } + else + { + return UserFormatStringError(node.MyLocation, + "failed to evaluate expression", $"'{expr}'"); + } + + case StringNode stringNode: + if (!string.IsNullOrEmpty(format)) + { + return UserFormatStringError(node.MyLocation, + "string items cannot specify format", $"'{format}'"); + } + else + { + return stringNode.Value; + } + + default: + return UserFormatStringError(node.MyLocation, + "invalid format item type", $"'{DiagnosticsHelpers.PrettyParamType(node.Type)}'"); + } + } + } +} diff --git a/ColorzCore/Lexer/Token.cs b/ColorzCore/Lexer/Token.cs index ba40c15..61eaa90 100644 --- a/ColorzCore/Lexer/Token.cs +++ b/ColorzCore/Lexer/Token.cs @@ -1,39 +1,41 @@ using ColorzCore.DataTypes; using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Lexer { public class Token { - public Location Location { get; } - public TokenType Type { get; } - public string FileName { get { return Location.file; } } - public int LineNumber { get { return Location.lineNum; } } - public int ColumnNumber { get { return Location.colNum; } } public string Content { get; } + public Location Location { get; } - public Token(TokenType type, string fileName, int lineNum, int colNum, string original = "") - { - Type = type; - Location = new Location(fileName, lineNum, colNum+1); - Content = original; - } + public string FileName => Location.file; + public int LineNumber => Location.line; + public int ColumnNumber => Location.column; - public Token(TokenType type, Location newLoc, string content) + public Token(TokenType type, Location location, string content = "") { Type = type; - this.Location = newLoc; + Location = location; Content = content; } - public override string ToString() + public override string ToString() => $"{Location}, {Type}: {Content}"; + + public Token MacroClone(MacroLocation macroLocation) => new Token(Type, Location.MacroClone(macroLocation), Content); + + // used for __LINE__ and __FILE__ + public Location GetSourceLocation() { - return String.Format("File {4}, Line {0}, Column {1}, {2}: {3}", LineNumber, ColumnNumber, Type, Content, FileName); + Location location = Location; + + while (location.macroLocation != null) + { + location = location.macroLocation.Location; + } + + return location; } } } diff --git a/ColorzCore/Lexer/TokenType.cs b/ColorzCore/Lexer/TokenType.cs index 96e8c4d..8a0911e 100644 --- a/ColorzCore/Lexer/TokenType.cs +++ b/ColorzCore/Lexer/TokenType.cs @@ -25,9 +25,21 @@ public enum TokenType LSHIFT_OP, RSHIFT_OP, SIGNED_RSHIFT_OP, + UNDEFINED_COALESCE_OP, + NOT_OP, AND_OP, XOR_OP, OR_OP, + LOGNOT_OP, + LOGAND_OP, + LOGOR_OP, + COMPARE_EQ, + COMPARE_NE, + COMPARE_LT, + COMPARE_LE, + COMPARE_GT, + COMPARE_GE, + ASSIGN, NUMBER, OPEN_BRACKET, CLOSE_BRACKET, diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 8e465f3..f780d28 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -5,6 +5,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using ColorzCore.DataTypes; using ColorzCore.IO; namespace ColorzCore.Lexer @@ -14,17 +15,17 @@ class Tokenizer public const int MAX_ID_LENGTH = 64; public static readonly Regex numRegex = new Regex("\\G([01]+b|0x[\\da-fA-F]+|\\$[\\da-fA-F]+|\\d+)"); public static readonly Regex idRegex = new Regex("\\G([a-zA-Z_][a-zA-Z0-9_]*)"); - public static readonly Regex stringRegex = new Regex("\\G(([^\\\"]|\\\\\\\")*)"); //"\\G(([^\\\\\\\"]|\\\\[rnt\\\\\\\"])*)"); - public static readonly Regex winPathnameRegex = new Regex(String.Format("\\G([^ \\{0}]|\\ |\\\\)+", Process(Path.GetInvalidPathChars()))); + public static readonly Regex stringRegex = new Regex(@"\G(([^\""]|\\\"")*)"); //"\\G(([^\\\\\\\"]|\\\\[rnt\\\\\\\"])*)"); + public static readonly Regex winPathnameRegex = new Regex(string.Format("\\G([^ \\{0}]|\\ |\\\\)+", Process(Path.GetInvalidPathChars()))); public static readonly Regex preprocDirectiveRegex = new Regex("\\G(#[a-zA-Z_][a-zA-Z0-9_]*)"); public static readonly Regex wordRegex = new Regex("\\G([^\\s]+)"); private static string Process(char[] chars) { StringBuilder sb = new StringBuilder(); - foreach(char c in chars) + foreach (char c in chars) { - switch(c) + switch (c) { case '.': case '\\': @@ -57,8 +58,8 @@ public Tokenizer() { multilineCommentNesting = 0; } - - public IEnumerable TokenizePhrase(string line, string fileName, int lineNum, int startOffs, int endOffs, int offset = 0) + + public IEnumerable TokenizePhrase(string line, int startOffs, int endOffs, Location location) { bool afterInclude = false, afterDirective = false, afterWhitespace = false; @@ -73,7 +74,8 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN multilineCommentNesting -= 1; curCol += 2; continue; - } else if (nextChar == '/' && curCol + 1 < endOffs && line[curCol + 1] == '*') + } + else if (nextChar == '/' && curCol + 1 < endOffs && line[curCol + 1] == '*') { multilineCommentNesting += 1; curCol += 2; @@ -85,7 +87,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN continue; } } - if (Char.IsWhiteSpace(nextChar) && nextChar != '\n') + if (char.IsWhiteSpace(nextChar) && nextChar != '\n') { curCol++; afterWhitespace = true; @@ -95,37 +97,44 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN switch (nextChar) { case ';': - yield return new Token(TokenType.SEMICOLON, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.SEMICOLON, location.OffsetBy(curCol)); break; case ':': - yield return new Token(TokenType.COLON, fileName, lineNum, curCol+offset); + if (curCol + 1 < endOffs && line[curCol + 1] == '=') // ':=' + { + yield return new Token(TokenType.ASSIGN, location.OffsetBy(curCol)); + curCol++; + break; + } + + yield return new Token(TokenType.COLON, location.OffsetBy(curCol)); break; case '{': - yield return new Token(TokenType.OPEN_BRACE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_BRACE, location.OffsetBy(curCol)); break; case '}': - yield return new Token(TokenType.CLOSE_BRACE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_BRACE, location.OffsetBy(curCol)); break; case '[': - yield return new Token(TokenType.OPEN_BRACKET, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_BRACKET, location.OffsetBy(curCol)); break; case ']': - yield return new Token(TokenType.CLOSE_BRACKET, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_BRACKET, location.OffsetBy(curCol)); break; case '(': - yield return new Token(TokenType.OPEN_PAREN, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_PAREN, location.OffsetBy(curCol)); break; case ')': - yield return new Token(TokenType.CLOSE_PAREN, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_PAREN, location.OffsetBy(curCol)); break; case '*': - yield return new Token(TokenType.MUL_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.MUL_OP, location.OffsetBy(curCol)); break; case '%': - yield return new Token(TokenType.MOD_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.MOD_OP, location.OffsetBy(curCol)); break; case ',': - yield return new Token(TokenType.COMMA, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.COMMA, location.OffsetBy(curCol)); break; case '/': if (curCol + 1 < endOffs && line[curCol + 1] == '/') @@ -141,11 +150,11 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } else { - yield return new Token(TokenType.DIV_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.DIV_OP, location.OffsetBy(curCol)); } break; case '+': - yield return new Token(TokenType.ADD_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.ADD_OP, location.OffsetBy(curCol)); break; case '-': if (afterWhitespace && afterDirective) @@ -154,45 +163,65 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (wsDelimited.Success) { string match = wsDelimited.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, IOUtility.UnescapePath(match)); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), IOUtility.UnescapePath(match)); curCol += match.Length; continue; } } - yield return new Token(TokenType.SUB_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.SUB_OP, location.OffsetBy(curCol)); break; case '&': - yield return new Token(TokenType.AND_OP, fileName, lineNum, curCol+offset); + if (curCol + 1 < endOffs && line[curCol + 1] == '&') + { + yield return new Token(TokenType.LOGAND_OP, location.OffsetBy(curCol)); + curCol++; + break; + } + + yield return new Token(TokenType.AND_OP, location.OffsetBy(curCol)); break; case '^': - yield return new Token(TokenType.XOR_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.XOR_OP, location.OffsetBy(curCol)); break; case '|': - yield return new Token(TokenType.OR_OP, fileName, lineNum, curCol+offset); + if (curCol + 1 < endOffs && line[curCol + 1] == '|') + { + yield return new Token(TokenType.LOGOR_OP, location.OffsetBy(curCol)); + curCol++; + break; + } + + yield return new Token(TokenType.OR_OP, location.OffsetBy(curCol)); break; case '\"': { curCol++; Match quoteInterior = stringRegex.Match(line, curCol, endOffs - curCol); string match = quoteInterior.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, /*IOUtility.UnescapeString(*/match/*)*/); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), /*IOUtility.UnescapeString(*/match/*)*/); curCol += match.Length; if (curCol == endOffs || line[curCol] != '\"') { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "Unclosed string."); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "Unclosed string."); } break; } case '<': if (curCol + 1 < endOffs && line[curCol + 1] == '<') { - yield return new Token(TokenType.LSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.LSHIFT_OP, location.OffsetBy(curCol)); + curCol++; + break; + } + else if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_LE, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "<"); + yield return new Token(TokenType.COMPARE_LT, location.OffsetBy(curCol)); break; } case '>': @@ -200,23 +229,68 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN { if (curCol + 2 < endOffs && line[curCol + 2] == '>') { - yield return new Token(TokenType.SIGNED_RSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.SIGNED_RSHIFT_OP, location.OffsetBy(curCol)); curCol += 2; } else { - yield return new Token(TokenType.RSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.RSHIFT_OP, location.OffsetBy(curCol)); curCol++; } break; } + else if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_GE, location.OffsetBy(curCol)); + curCol++; + break; + } + else + { + yield return new Token(TokenType.COMPARE_GT, location.OffsetBy(curCol)); + break; + } + case '=': + if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_EQ, location.OffsetBy(curCol)); + curCol++; + break; + } + else + { + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "="); + break; + } + case '!': + if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_NE, location.OffsetBy(curCol)); + curCol++; + break; + } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, ">"); + yield return new Token(TokenType.LOGNOT_OP, location.OffsetBy(curCol)); + break; + } + case '~': + yield return new Token(TokenType.NOT_OP, location.OffsetBy(curCol)); + break; + case '?': + if (curCol + 1 < endOffs && line[curCol + 1] == '?') + { + yield return new Token(TokenType.UNDEFINED_COALESCE_OP, location.OffsetBy(curCol)); + curCol++; + break; + } + else + { + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "?"); break; } case '\n': - yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.NEWLINE, location.OffsetBy(curCol)); break; default: if (afterInclude) @@ -225,7 +299,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (winPath.Success) { string match = winPath.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, IOUtility.UnescapePath(match)); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), IOUtility.UnescapePath(match)); curCol += match.Length; afterInclude = false; continue; @@ -242,14 +316,14 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN int idCol = curCol; curCol += match.Length; if (curCol < endOffs && line[curCol] == '(') - yield return new Token(TokenType.MAYBE_MACRO, fileName, lineNum, idCol, match); - else - yield return new Token(TokenType.IDENTIFIER, fileName, lineNum, idCol, match); - if (curCol < endOffs && (Char.IsLetterOrDigit(line[curCol]) | line[curCol] == '_')) + yield return new Token(TokenType.MAYBE_MACRO, location.OffsetBy(idCol), match); + else + yield return new Token(TokenType.IDENTIFIER, location.OffsetBy(idCol), match); + if (curCol < endOffs && (char.IsLetterOrDigit(line[curCol]) | line[curCol] == '_')) { Match idMatch2 = new Regex("[a-zA-Z0-9_]+").Match(line, curCol, endOffs - curCol); match = idMatch2.Value; - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, String.Format("Identifier longer than {0} characters.", MAX_ID_LENGTH)); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), $"Identifier longer than {MAX_ID_LENGTH} characters."); curCol += match.Length; } continue; @@ -259,9 +333,9 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN { string match = numMatch.Value; //Verify that next token isn't start of an identifier - if (curCol + match.Length >= endOffs || (!Char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) + if (curCol + match.Length >= endOffs || (!char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) { - yield return new Token(TokenType.NUMBER, fileName, lineNum, curCol, match.TrimEnd()); + yield return new Token(TokenType.NUMBER, location.OffsetBy(curCol), match.TrimEnd()); curCol += match.Length; continue; } @@ -270,7 +344,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (directiveMatch.Success) { string match = directiveMatch.Value; - yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, fileName, lineNum, curCol, match); + yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, location.OffsetBy(curCol), match); curCol += match.Length; if (match.Substring(1).Equals("include") || match.Substring(1).Equals("incbin")) { @@ -281,7 +355,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } } string restOfWord = new Regex("\\G\\S+").Match(line, curCol, endOffs - curCol).Value; - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, restOfWord); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), restOfWord); curCol += restOfWord.Length; continue; } @@ -290,9 +364,15 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN afterWhitespace = false; } } - public IEnumerable TokenizeLine(string line, string fileName, int lineNum, int offset = 0) + + public IEnumerable TokenizePhrase(string line, Location location) + { + return TokenizePhrase(line, 0, line.Length, location); + } + + public static IEnumerable TokenizeLine(string line, Location location) { - return TokenizePhrase(line, fileName, lineNum, 0, line.Length, offset); + return new Tokenizer().TokenizePhrase(line, 0, line.Length, location); } /*** @@ -305,29 +385,35 @@ public IEnumerable Tokenize(Stream input, string fileName) int curLine = 1; while (!sin.EndOfStream) { - string line = sin.ReadLine(); - + string line = sin.ReadLine()!; + //allow escaping newlines - while (line.Length > 0 && line.Substring(line.Length-1) == "\\") + while (line.Length > 0 && line[line.Length - 1] == '\\') { curLine++; line = line.Substring(0, line.Length - 1) + " " + sin.ReadLine(); } - - foreach (Token t in TokenizeLine(line, fileName, curLine)) + + Location location = new Location(fileName, curLine, 1); + foreach (Token t in TokenizePhrase(line, location)) { yield return t; } - yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); + yield return new Token(TokenType.NEWLINE, location.OffsetBy(line.Length)); curLine++; } } - public IEnumerable Tokenize(FileStream fs) + public IEnumerable TokenizeFile(FileStream fs, string filename) { - foreach (Token t in Tokenize(fs, fs.Name)) + foreach (Token t in Tokenize(fs, filename)) yield return t; fs.Close(); } + + public IEnumerable Tokenize(FileStream fs) + { + return TokenizeFile(fs, fs.Name); + } } } diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index 8998bb6..deff126 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -1,9 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; namespace ColorzCore.Parser.AST @@ -12,27 +11,16 @@ public abstract class AtomNodeKernel : IAtomNode { public abstract int Precedence { get; } - public ParamType Type { get { return ParamType.ATOM; } } - - public virtual Maybe GetIdentifier() - { - return new Nothing(); - } + public ParamType Type => ParamType.ATOM; public abstract string PrettyPrint(); - public abstract IEnumerable ToTokens(); public abstract Location MyLocation { get; } - public abstract Maybe TryEvaluate(TAction handler); - - public IParamNode SimplifyExpressions(TAction handler) - { - return this.Simplify(handler); - } + public abstract int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); - public Maybe AsAtom() + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { - return new Just(this); + return this.Simplify(handler, evaluationPhase); } } } diff --git a/ColorzCore/Parser/AST/BlockNode.cs b/ColorzCore/Parser/AST/BlockNode.cs deleted file mode 100644 index f122600..0000000 --- a/ColorzCore/Parser/AST/BlockNode.cs +++ /dev/null @@ -1,57 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.IO; -using ColorzCore.Lexer; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.AST -{ - class BlockNode : ILineNode - { - public IList Children { get; } - - public int Size { - get - { - return Children.Sum((ILineNode n) => n.Size); - } } - - public BlockNode() - { - Children = new List(); - } - - public string PrettyPrint(int indentation) - { - StringBuilder sb = new StringBuilder(); - sb.Append(' ', indentation); - sb.Append('{'); - sb.Append('\n'); - foreach(ILineNode i in Children) - { - sb.Append(i.PrettyPrint(indentation + 4)); - sb.Append('\n'); - } - sb.Append(' ', indentation); - sb.Append('}'); - return sb.ToString(); - } - - public void WriteData(IOutput output) - { - foreach(ILineNode child in Children) - { - child.WriteData(output); - } - } - - public void EvaluateExpressions(ICollection undefinedIdentifiers) - { - foreach (ILineNode line in Children) - line.EvaluateExpressions(undefinedIdentifiers); - } - } -} diff --git a/ColorzCore/Parser/AST/DataNode.cs b/ColorzCore/Parser/AST/DataNode.cs index ca85cd0..e0d6af2 100644 --- a/ColorzCore/Parser/AST/DataNode.cs +++ b/ColorzCore/Parser/AST/DataNode.cs @@ -1,17 +1,16 @@ -using ColorzCore.IO; -using ColorzCore.Lexer; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { class DataNode : ILineNode { - private int offset; - private byte[] data; + private readonly int offset; + private readonly byte[] data; public DataNode(int offset, byte[] data) { @@ -23,16 +22,16 @@ public DataNode(int offset, byte[] data) public string PrettyPrint(int indentation) { - return String.Format("Raw Data Block of Length {0}", Size); + return $"Raw Data Block of Length {Size}"; } - + public void WriteData(IOutput output) { output.WriteTo(offset, data); } - public void EvaluateExpressions(ICollection undefinedIdentifiers) - { + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) + { // Nothing to be done because we contain no expressions. } } diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 9311846..262ceb9 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -1,37 +1,34 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { public interface IAtomNode : IParamNode { //TODO: Simplify() partial evaluation as much as is defined, to save on memory space. - int Precedence { get; } - Maybe GetIdentifier(); - IEnumerable ToTokens(); - Maybe TryEvaluate(TAction handler); //Simplifies the AST as much as possible. + int Precedence { get; } + int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); //Simplifies the AST as much as possible. } public static class AtomExtensions { public static int CoerceInt(this IAtomNode n) { - return n.TryEvaluate((Exception e) => { throw e; }).FromJust; + return n.TryEvaluate(e => throw e, EvaluationPhase.Final)!.Value; } - public static IAtomNode Simplify(this IAtomNode n, TAction handler) + + public static IAtomNode Simplify(this IAtomNode self, Action handler, EvaluationPhase evaluationPhase) { - IAtomNode ret = n; - n.TryEvaluate(handler).IfJust((int i) => { ret = FromInt(n.MyLocation, i); }); - return ret; + return self.TryEvaluate(handler, evaluationPhase).IfJust(intValue => FromInt(self.MyLocation, intValue), () => self); } - public static IAtomNode FromInt(Location l, int i) + + public static IAtomNode FromInt(Location location, int intValue) { - return new NumberNode(l, i); + return new NumberNode(location, intValue); } } } diff --git a/ColorzCore/Parser/AST/ILineNode.cs b/ColorzCore/Parser/AST/ILineNode.cs index c92f331..7544a00 100644 --- a/ColorzCore/Parser/AST/ILineNode.cs +++ b/ColorzCore/Parser/AST/ILineNode.cs @@ -1,18 +1,17 @@ -using ColorzCore.IO; -using ColorzCore.Lexer; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { - interface ILineNode + public interface ILineNode { int Size { get; } string PrettyPrint(int indentation); void WriteData(IOutput output); - void EvaluateExpressions(ICollection undefinedIdentifiers); //Return: undefined identifiers + void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase); // outputs errors into evaluationErrors } } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 7c70c0b..b0f6336 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -1,28 +1,27 @@ -using ColorzCore.DataTypes; -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.Lexer; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; namespace ColorzCore.Parser.AST { public interface IParamNode { - string ToString(); //For use in other programs. + string? ToString(); //For use in other programs. ParamType Type { get; } string PrettyPrint(); Location MyLocation { get; } - IParamNode SimplifyExpressions(TAction handler); //TODO: Abstract this into a general traverse method. - Maybe AsAtom(); + + // TODO: Abstract this into a general traverse method. + IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase); } public static class ParamExtensions { - public static IParamNode Simplify(this IParamNode n) + public static IParamNode Simplify(this IParamNode n, EvaluationPhase evaluationPhase) { - return n.SimplifyExpressions((Exception e) => { }); + return n.SimplifyExpressions(e => { }, evaluationPhase); } } } diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index c0db903..56ca345 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -8,74 +9,96 @@ namespace ColorzCore.Parser.AST { public class IdentifierNode : AtomNodeKernel { - private Token identifier; - readonly ImmutableStack scope; + private readonly ImmutableStack boundScope; - public override int Precedence { get { return 11; } } - public override Location MyLocation { get { return identifier.Location; } } + public Token IdentifierToken { get; } - public IdentifierNode(Token id, ImmutableStack scopes) - { - identifier = id; - scope = scopes; - } - - private int ToInt() + public override int Precedence => int.MaxValue; + public override Location MyLocation => IdentifierToken.Location; + + public IdentifierNode(Token identifierToken, ImmutableStack scope) + { + IdentifierToken = identifierToken; + boundScope = scope; + } + + private int ToInt(EvaluationPhase evaluationPhase) { - ImmutableStack temp = scope; - while(!temp.IsEmpty) + if (boundScope != null) { - if(temp.Head.HasLocalLabel(identifier.Content)) - return temp.Head.GetLabel(identifier.Content); + if (evaluationPhase == EvaluationPhase.Early) + { + /* We only check the immediate scope. + * As an outer scope symbol may get shadowed by an upcoming inner scope symbol. */ + + if (boundScope.Head.HasLocalSymbol(IdentifierToken.Content)) + { + return boundScope.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + } + } else - temp = temp.Tail; + { + for (ImmutableStack it = boundScope; !it.IsEmpty; it = it.Tail) + { + if (it.Head.HasLocalSymbol(IdentifierToken.Content)) + { + return it.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + } + } + } } - throw new UndefinedIdentifierException(identifier); + + throw new UndefinedIdentifierException(IdentifierToken); } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { try { - return new Just(ToInt()); - } catch(UndefinedIdentifierException e) + return ToInt(evaluationPhase); + } + catch (UndefinedIdentifierException e) { handler(e); - return new Nothing(); + return null; + } + catch (Closure.SymbolComputeException e) + { + handler(e); + return null; } - } - - public override Maybe GetIdentifier() - { - return new Just(identifier.Content); } public override string PrettyPrint() { try { - return "0x"+ToInt().ToString("X"); + return $"0x{ToInt(EvaluationPhase.Immediate):X}"; } catch (UndefinedIdentifierException) { - return identifier.Content; + return IdentifierToken.Content; + } + catch (Closure.SymbolComputeException) + { + return IdentifierToken.Content; } } - public override IEnumerable ToTokens() { yield return identifier; } + public override string ToString() + { + return IdentifierToken.Content; + } + // TODO: move this outside of this class public class UndefinedIdentifierException : Exception { public Token CausedError { get; set; } - public UndefinedIdentifierException(Token causedError) : base("Undefined identifier: " + causedError.Content) + + public UndefinedIdentifierException(Token causedError) : base($"Undefined identifier `{causedError.Content}`") { - this.CausedError = causedError; + CausedError = causedError; } } - - public override string ToString() - { - return identifier.Content; - } } } diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 92ef528..82a3fa6 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -1,10 +1,10 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { @@ -25,10 +25,10 @@ public override string ToString() { StringBuilder sb = new StringBuilder(); sb.Append('['); - for(int i=0; i ToTokens() - { - //Similar code to ParenthesizedAtom - IList> temp = new List>(); - foreach(IAtomNode n in Interior) - { - temp.Add(new List(n.ToTokens())); - } - Location myStart = MyLocation; - Location myEnd = temp.Count > 0 ? temp.Last().Last().Location : MyLocation; - yield return new Token(TokenType.OPEN_BRACKET, new Location(myStart.file, myStart.lineNum, myStart.colNum - 1), "["); - for (int i = 0; i < temp.Count; i++) - { - foreach (Token t in temp[i]) - { - yield return t; - } - if (i < temp.Count - 1) - { - Location tempEnd = temp[i].Last().Location; - yield return new Token(TokenType.COMMA, new Location(tempEnd.file, tempEnd.lineNum, tempEnd.colNum + temp[i].Last().Content.Length), ","); - } - } - yield return new Token(TokenType.CLOSE_BRACKET, new Location(myEnd.file, myEnd.lineNum, myEnd.colNum + (temp.Count > 0 ? temp.Last().Last().Content.Length : 1)), "]"); - } - - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } - public IParamNode SimplifyExpressions(TAction handler) + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { IEnumerable acc = new List(); - for(int i=0; i AsAtom() { return new Nothing(); } } } diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index 65e42b2..5d7668a 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -8,6 +9,7 @@ namespace ColorzCore.Parser.AST { + // TODO: what do we need this for? class MacroInvocationNode : IParamNode { public class MacroException : Exception @@ -21,15 +23,13 @@ public MacroException(MacroInvocationNode min) : base(min.invokeToken.Content) private readonly EAParser p; private readonly Token invokeToken; - private readonly ImmutableStack scope; public IList> Parameters { get; } - public MacroInvocationNode(EAParser p, Token invokeTok, IList> parameters, ImmutableStack scopes) + public MacroInvocationNode(EAParser p, Token invokeTok, IList> parameters) { this.p = p; - this.invokeToken = invokeTok; - this.Parameters = parameters; - this.scope = scopes; + invokeToken = invokeTok; + Parameters = parameters; } public ParamType Type => ParamType.MACRO; @@ -39,7 +39,7 @@ public string PrettyPrint() StringBuilder sb = new StringBuilder(); sb.Append(invokeToken.Content); sb.Append('('); - for(int i=0; i ExpandMacro() - { - return p.Macros.GetMacro(invokeToken.Content, Parameters.Count).ApplyMacro(invokeToken, Parameters, scope); - } - - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } - - public string Name { get { return invokeToken.Content; } } - public Location MyLocation { get { return invokeToken.Location; } } - public Maybe AsAtom() { return new Nothing(); } - - public IParamNode SimplifyExpressions(TAction handler) + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { handler(new MacroException(this)); return this; diff --git a/ColorzCore/Parser/AST/NegationNode.cs b/ColorzCore/Parser/AST/NegationNode.cs deleted file mode 100644 index 43ab565..0000000 --- a/ColorzCore/Parser/AST/NegationNode.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.DataTypes; -using ColorzCore.Lexer; - -namespace ColorzCore.Parser.AST -{ - class NegationNode : AtomNodeKernel - { - private Token myToken; - private IAtomNode interior; - - public NegationNode(Token token, IAtomNode inside) - { - myToken = token; - interior = inside; - } - - public override int Precedence => 11; - public override Location MyLocation => myToken.Location; - - public override string PrettyPrint() - { - return '-' + interior.PrettyPrint(); - } - - public override IEnumerable ToTokens() - { - yield return myToken; - foreach(Token t in interior.ToTokens()) - yield return t; - } - - public override Maybe TryEvaluate(TAction handler) - { - return interior.TryEvaluate(handler).Fmap((int x) => -x); - } - } -} diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index d8bcca4..ed64eb9 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -2,45 +2,42 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; namespace ColorzCore.Parser.AST { public class NumberNode : AtomNodeKernel { - private int value; + public Token SourceToken { get; } + public int Value { get; } - public override Location MyLocation { get; } - public override int Precedence { get { return 11; } } + public override Location MyLocation => SourceToken.Location; + public override int Precedence => int.MaxValue; - public NumberNode(Token num) - { - MyLocation = num.Location; - value = num.Content.ToInt(); + public NumberNode(Token num) + { + SourceToken = num; + Value = num.Content.ToInt(); } + public NumberNode(Token text, int value) { - MyLocation = text.Location; - this.value = value; + SourceToken = text; + Value = value; } + public NumberNode(Location loc, int value) { - MyLocation = loc; - this.value = value; + SourceToken = new Token(TokenType.NUMBER, loc, value.ToString()); + Value = value; } - - public override IEnumerable ToTokens () { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { - return new Just(value); + return Value; } - public override string PrettyPrint() - { - return "0x" + value.ToString("X"); - } + public override string PrettyPrint() => Value >= 16 ? $"0x{Value:X}" : $"{Value}"; } } diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 65b751c..3d02f17 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -1,107 +1,145 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; -using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { - delegate int BinaryIntOp(int a, int b); - - class OperatorNode : AtomNodeKernel + public class OperatorNode : AtomNodeKernel { - public static readonly Dictionary Operators = new Dictionary { - { TokenType.MUL_OP , (x, y) => x*y }, - { TokenType.DIV_OP , (x, y) => x/y }, - { TokenType.MOD_OP , (x, y) => x%y }, - { TokenType.ADD_OP , (x, y) => x+y }, - { TokenType.SUB_OP , (x, y) => x-y }, - { TokenType.LSHIFT_OP , (x, y) => x< (int)(((uint)x)>>y) }, - { TokenType.SIGNED_RSHIFT_OP , (x, y) => x>>y }, - { TokenType.AND_OP , (x, y) => x&y }, - { TokenType.XOR_OP , (x, y) => x^y }, - { TokenType.OR_OP , (x, y) => x|y } - }; - - private IAtomNode left, right; - private Token op; - public override int Precedence { get; } - - public override Location MyLocation { get { return op.Location; } } - - public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) - { - left = l; - right = r; - this.op = op; + public IAtomNode Left { get; private set; } + public IAtomNode Right { get; private set; } + public Token OperatorToken { get; } + + public override int Precedence { get; } + public override Location MyLocation => OperatorToken.Location; + + public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) + { + Left = l; + Right = r; + OperatorToken = op; Precedence = prec; - } + } + + public string OperatorString => OperatorToken.Type switch + { + TokenType.MUL_OP => "*", + TokenType.DIV_OP => "/", + TokenType.MOD_OP => "%", + TokenType.ADD_OP => "+", + TokenType.SUB_OP => "-", + TokenType.LSHIFT_OP => "<<", + TokenType.RSHIFT_OP => ">>", + TokenType.SIGNED_RSHIFT_OP => ">>>", + TokenType.UNDEFINED_COALESCE_OP => "??", + TokenType.AND_OP => "&", + TokenType.XOR_OP => "^", + TokenType.OR_OP => "|", + TokenType.LOGAND_OP => "&&", + TokenType.LOGOR_OP => "||", + TokenType.COMPARE_EQ => "==", + TokenType.COMPARE_NE => "!=", + TokenType.COMPARE_LT => "<", + TokenType.COMPARE_LE => "<=", + TokenType.COMPARE_GT => ">", + TokenType.COMPARE_GE => ">=", + _ => "" + }; public override string PrettyPrint() { - StringBuilder sb = new StringBuilder(left.PrettyPrint()); - switch(op.Type) - { - case TokenType.MUL_OP: - sb.Append("*"); - break; - case TokenType.DIV_OP: - sb.Append("/"); - break; - case TokenType.ADD_OP: - sb.Append("+"); - break; - case TokenType.SUB_OP: - sb.Append("-"); - break; - case TokenType.LSHIFT_OP: - sb.Append("<<"); - break; - case TokenType.RSHIFT_OP: - sb.Append(">>"); - break; - case TokenType.SIGNED_RSHIFT_OP: - sb.Append(">>>"); - break; - case TokenType.AND_OP: - sb.Append("&"); - break; - case TokenType.XOR_OP: - sb.Append("^"); - break; - case TokenType.OR_OP: - sb.Append("|"); - break; - default: - break; - } - sb.Append(right.PrettyPrint()); - return sb.ToString(); + return $"({Left.PrettyPrint()} {OperatorString} {Right.PrettyPrint()})"; } - public override IEnumerable ToTokens() + + private int? TryCoalesceUndefined(Action handler) { - foreach(Token t in left.ToTokens()) + List? leftExceptions = null; + + // the left side of an undefined coalescing operation is allowed to raise exactly UndefinedIdentifierException + // we need to catch that, so don't forward all exceptions raised by left just yet + + int? leftValue = Left.TryEvaluate(e => (leftExceptions ??= new List()).Add(e), EvaluationPhase.Final); + + if (leftExceptions == null) { - yield return t; + // left evaluated properly => result is left + return leftValue; } - yield return op; - foreach(Token t in right.ToTokens()) + else if (leftExceptions.All(e => e is IdentifierNode.UndefinedIdentifierException)) { - yield return t; + // left did not evalute due to undefined identifier => result is right + return Right.TryEvaluate(handler, EvaluationPhase.Final); + } + else + { + // left failed to evaluate for some other reason + foreach (Exception e in leftExceptions.Where(e => e is not IdentifierNode.UndefinedIdentifierException)) + { + handler(e); + } + + return null; } } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { - Maybe l = left.TryEvaluate(handler); - l.IfJust((int i) => { this.left = new NumberNode(left.MyLocation, i); }); - Maybe r = right.TryEvaluate(handler); - r.IfJust((int i) => { this.right = new NumberNode(right.MyLocation, i); }); - return l.Bind((int newL) => r.Fmap((int newR) => Operators[op.Type](newL, newR))); + /* undefined-coalescing operator is special because + * 1. it should not be evaluated early. + * 2. it is legal for its left operand to fail evaluation. */ + if (OperatorToken.Type == TokenType.UNDEFINED_COALESCE_OP) + { + // TODO: better exception types here? + + switch (evaluationPhase) + { + case EvaluationPhase.Early: + /* NOTE: maybe one could optimize this by reducing this if left can be evaluated early? */ + handler(new Exception("The value of a '??' expression cannot be resolved early.")); + return null; + + default: + return TryCoalesceUndefined(handler); + } + } + + int? leftValue = Left.TryEvaluate(handler, evaluationPhase); + leftValue.IfJust(i => Left = new NumberNode(Left.MyLocation, i)); + + int? rightValue = Right.TryEvaluate(handler, evaluationPhase); + rightValue.IfJust(i => Right = new NumberNode(Right.MyLocation, i)); + + if (leftValue is int lhs && rightValue is int rhs) + { + return OperatorToken.Type switch + { + TokenType.MUL_OP => lhs * rhs, + TokenType.DIV_OP => lhs / rhs, + TokenType.MOD_OP => lhs % rhs, + TokenType.ADD_OP => lhs + rhs, + TokenType.SUB_OP => lhs - rhs, + TokenType.LSHIFT_OP => lhs << rhs, + TokenType.RSHIFT_OP => (int)(((uint)lhs) >> rhs), + TokenType.SIGNED_RSHIFT_OP => lhs >> rhs, + TokenType.AND_OP => lhs & rhs, + TokenType.XOR_OP => lhs ^ rhs, + TokenType.OR_OP => lhs | rhs, + TokenType.LOGAND_OP => lhs != 0 ? rhs : 0, + TokenType.LOGOR_OP => lhs != 0 ? lhs : rhs, + TokenType.COMPARE_EQ => lhs == rhs ? 1 : 0, + TokenType.COMPARE_NE => lhs != rhs ? 1 : 0, + TokenType.COMPARE_LT => lhs < rhs ? 1 : 0, + TokenType.COMPARE_LE => lhs <= rhs ? 1 : 0, + TokenType.COMPARE_GE => lhs >= rhs ? 1 : 0, + TokenType.COMPARE_GT => lhs > rhs ? 1 : 0, + _ => null, + }; + } + + return null; } } } diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index 98e3018..5005f06 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -1,34 +1,48 @@ -using ColorzCore.IO; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Raws; using System; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { - class RawNode : StatementNode + public class RawNode : ILineNode { - private Raw myRaw; - private Token myToken; - private int offset; + public IList Parameters { get; } + public Raw Raw { get; } + private int Offset { get; } - public RawNode(Raw raw, Token t, int offset, IList paramList) : base(paramList) + public RawNode(Raw raw, int offset, IList parameters) { - myToken = t; - myRaw = raw; - this.offset = offset; + Parameters = parameters; + Raw = raw; + Offset = offset; } - public override int Size => myRaw.LengthBytes(Parameters.Count); + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) + { + for (int i = 0; i < Parameters.Count; i++) + { + Parameters[i] = Parameters[i].SimplifyExpressions(e => evaluationErrors.Add(e switch + { + IdentifierNode.UndefinedIdentifierException uie => (uie.CausedError.Location, uie), + Closure.SymbolComputeException sce => (sce.Expression.MyLocation, sce), + _ => (Parameters[i].MyLocation, e), + }), evaluationPhase); + } + } + + public int Size => Raw.LengthBytes(Parameters.Count); - public override string PrettyPrint(int indentation) + public string PrettyPrint(int indentation) { StringBuilder sb = new StringBuilder(); sb.Append(' ', indentation); - sb.Append(myToken.Content); + sb.Append(Raw.Name); foreach (IParamNode n in Parameters) { sb.Append(' '); @@ -36,9 +50,10 @@ public override string PrettyPrint(int indentation) } return sb.ToString(); } - public override void WriteData(IOutput output) + + public void WriteData(IOutput output) { - output.WriteTo(offset, myRaw.GetBytes(Parameters)); + output.WriteTo(Offset, Raw.GetBytes(Parameters)); } } } diff --git a/ColorzCore/Parser/AST/StatementNode.cs b/ColorzCore/Parser/AST/StatementNode.cs deleted file mode 100644 index abbf331..0000000 --- a/ColorzCore/Parser/AST/StatementNode.cs +++ /dev/null @@ -1,44 +0,0 @@ -using ColorzCore.IO; -using ColorzCore.Lexer; -using ColorzCore.Raws; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.AST -{ - abstract class StatementNode : ILineNode - { - public IList Parameters { get; } - - protected StatementNode(IList parameters) - { - Parameters = parameters; - } - - public abstract int Size { get; } - - public abstract string PrettyPrint(int indentation); - public abstract void WriteData(IOutput output); - - public void EvaluateExpressions(ICollection undefinedIdentifiers) - { - for (int i = 0; i < Parameters.Count; i++) - Parameters[i] = Parameters[i].SimplifyExpressions((Exception e) => - { - try - { - throw e; - } - catch (IdentifierNode.UndefinedIdentifierException uie) - { - undefinedIdentifiers.Add(uie.CausedError); - } - catch (Exception) - { } - }); - } - } -} diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index e9effae..eb918be 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -11,48 +12,28 @@ namespace ColorzCore.Parser.AST { public class StringNode : IParamNode { - public static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); - public Token MyToken { get; } + public Token SourceToken { get; } - public Location MyLocation { get { return MyToken.Location; } } - public ParamType Type { get { return ParamType.STRING; } } + public Location MyLocation => SourceToken.Location; + public ParamType Type => ParamType.STRING; - public StringNode(Token value) - { - MyToken = value; - } + public string Value => SourceToken.Content; - public IEnumerable ToBytes() + public StringNode(Token value) { - return Encoding.ASCII.GetBytes(ToString()); + SourceToken = value; } public override string ToString() { - return MyToken.Content; - } - public string PrettyPrint() - { - return '"' + ToString() + '"'; + return Value; } - public IEnumerable ToTokens() { yield return MyToken; } - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } - - public bool IsValidIdentifier() - { - return idRegex.IsMatch(MyToken.Content); - } - public IdentifierNode ToIdentifier(ImmutableStack scope) + public string PrettyPrint() { - return new IdentifierNode(MyToken, scope); + return $"\"{Value}\""; } - public Maybe AsAtom() { return new Nothing(); } - - public IParamNode SimplifyExpressions(TAction handler) { return this; } + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) => this; } } diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs new file mode 100644 index 0000000..2f15f30 --- /dev/null +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -0,0 +1,62 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.Lexer; + +namespace ColorzCore.Parser.AST +{ + public class UnaryOperatorNode : AtomNodeKernel + { + public IAtomNode Inner { get; private set; } + + public Token OperatorToken { get; } + + public UnaryOperatorNode(Token token, IAtomNode inside) + { + OperatorToken = token; + Inner = inside; + } + + public override int Precedence => int.MaxValue; + public override Location MyLocation => OperatorToken.Location; + + public string OperatorString => OperatorToken.Type switch + { + TokenType.SUB_OP => "-", + TokenType.NOT_OP => "~", + TokenType.LOGNOT_OP => "!", + _ => "", + }; + + public override string PrettyPrint() + { + return OperatorString + Inner.PrettyPrint(); + } + + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) + { + int? inner = Inner.TryEvaluate(handler, evaluationPhase); + + if (inner != null) + { + // int? is magic and all of these operations conveniently propagate nulls + + return OperatorToken.Type switch + { + TokenType.SUB_OP => -inner, + TokenType.NOT_OP => ~inner, + TokenType.LOGNOT_OP => inner != 0 ? 0 : 1, + _ => null, + }; + } + else + { + return null; + } + } + } +} diff --git a/ColorzCore/Parser/AtomParser.cs b/ColorzCore/Parser/AtomParser.cs new file mode 100644 index 0000000..312f54b --- /dev/null +++ b/ColorzCore/Parser/AtomParser.cs @@ -0,0 +1,262 @@ +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public static class AtomParser + { + private static readonly IDictionary precedences = new Dictionary { + { TokenType.MUL_OP, 3 }, + { TokenType.DIV_OP, 3 }, + { TokenType.MOD_OP, 3 }, + { TokenType.ADD_OP, 4 }, + { TokenType.SUB_OP, 4 }, + { TokenType.LSHIFT_OP, 5 }, + { TokenType.RSHIFT_OP, 5 }, + { TokenType.SIGNED_RSHIFT_OP, 5 }, + { TokenType.COMPARE_GE, 6 }, + { TokenType.COMPARE_GT, 6 }, + { TokenType.COMPARE_LT, 6 }, + { TokenType.COMPARE_LE, 6 }, + { TokenType.COMPARE_EQ, 7 }, + { TokenType.COMPARE_NE, 7 }, + { TokenType.AND_OP, 8 }, + { TokenType.XOR_OP, 9 }, + { TokenType.OR_OP, 10 }, + { TokenType.LOGAND_OP, 11 }, + { TokenType.LOGOR_OP, 12 }, + { TokenType.UNDEFINED_COALESCE_OP, 13 }, + }; + + public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); + + public static IAtomNode? ParseAtom(this EAParser self, MergeableGenerator tokens) + { + //Use Shift Reduce Parsing + Token localHead = tokens.Current; + Stack> grammarSymbols = new Stack>(); + bool ended = false; + while (!ended) + { + bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; + Token lookAhead = tokens.Current; + + if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. + { + //Verify next symbol to be a binary operator. + switch (lookAhead.Type) + { + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.SUB_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + if (precedences.TryGetValue(lookAhead.Type, out int precedence)) + { + self.Reduce(grammarSymbols, precedence); + } + shift = true; + break; + case TokenType.UNDEFINED_COALESCE_OP: + // '??' is right-associative, so don't reduce here + shift = true; + break; + default: + ended = true; + break; + } + } + else if (!ended) //Is just an operator. Error if two operators in a row. + { + //Error if two operators in a row. + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.NUMBER: + shift = true; + break; + case TokenType.OPEN_PAREN: + { + tokens.MoveNext(); + IAtomNode? interior = self.ParseAtom(tokens); + if (tokens.Current.Type != TokenType.CLOSE_PAREN) + { + self.Logger.Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); + return null; + } + else if (interior == null) + { + self.Logger.Error(lookAhead.Location, "Expected expression inside paretheses. "); + return null; + } + else + { + grammarSymbols.Push(new Left(interior)); + tokens.MoveNext(); + break; + } + } + case TokenType.SUB_OP: + case TokenType.LOGNOT_OP: + case TokenType.NOT_OP: + { + //Assume unary negation. + tokens.MoveNext(); + IAtomNode? interior = self.ParseAtom(tokens); + if (interior == null) + { + self.Logger.Error(lookAhead.Location, "Expected expression after unary operator."); + return null; + } + grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); + break; + } + case TokenType.COMMA: + self.Logger.Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); + self.IgnoreRestOfStatement(tokens); + return null; + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + case TokenType.UNDEFINED_COALESCE_OP: + default: + self.Logger.Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); + self.IgnoreRestOfStatement(tokens); + return null; + } + } + + if (shift) + { + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + if (self.ExpandIdentifier(tokens, true)) + { + continue; + } + + grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch + { + "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), + _ => self.BindIdentifier(lookAhead), + })); + + break; + + case TokenType.MAYBE_MACRO: + self.ExpandIdentifier(tokens, true); + continue; + case TokenType.NUMBER: + grammarSymbols.Push(new Left(new NumberNode(lookAhead))); + break; + case TokenType.ERROR: + self.Logger.Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); + tokens.MoveNext(); + return null; + default: + grammarSymbols.Push(new Right(lookAhead)); + break; + } + tokens.MoveNext(); + continue; + } + } + while (grammarSymbols.Count > 1) + { + self.Reduce(grammarSymbols, int.MaxValue); + } + if (grammarSymbols.Peek().IsRight) + { + self.Logger.Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); + } + return grammarSymbols.Peek().GetLeft; + } + + /*** + * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd + * the precedences of the IAtomNodes is increasing. + * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. + * + */ + private static void Reduce(this EAParser self, Stack> grammarSymbols, int targetPrecedence) + { + while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) + { + // These shouldn't error... + IAtomNode r = grammarSymbols.Pop().GetLeft; + + if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) + { + grammarSymbols.Push(new Left(r)); + break; + } + else + { + Token op = grammarSymbols.Pop().GetRight; + IAtomNode l = grammarSymbols.Pop().GetLeft; + + OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UnintuitiveExpressionMacros)) + { + if (DiagnosticsHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) + { + MacroLocation? mloc = operatorNode.MyLocation.macroLocation; + string message = DiagnosticsHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); + + if (mloc != null) + { + message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; + } + else + { + message += "\nUnintuitive expression resulting from expansion of macro."; + } + + message += "\nConsider guarding your expressions using parenthesis."; + + self.Logger.Warning(operatorNode.MyLocation, message); + } + } + + grammarSymbols.Push(new Left(operatorNode)); + } + } + } + } +} diff --git a/ColorzCore/Parser/BaseClosure.cs b/ColorzCore/Parser/BaseClosure.cs deleted file mode 100644 index d950c8f..0000000 --- a/ColorzCore/Parser/BaseClosure.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace ColorzCore.Parser -{ - class BaseClosure : Closure - { - private EAParser enclosing; - public BaseClosure(EAParser enclosing) - { - this.enclosing = enclosing; - } - public override bool HasLocalLabel(string label) - { - return label.ToUpper() == "CURRENTOFFSET" || base.HasLocalLabel(label); - } - } -} diff --git a/ColorzCore/Parser/Closure.cs b/ColorzCore/Parser/Closure.cs deleted file mode 100644 index 452f8ea..0000000 --- a/ColorzCore/Parser/Closure.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.Collections.Generic; - -namespace ColorzCore.Parser -{ - public class Closure - { - private Dictionary Labels { get; } - - public Closure() - { - Labels = new Dictionary(); - } - public virtual bool HasLocalLabel(string label) - { - return Labels.ContainsKey(label); - } - public virtual int GetLabel(string label) - { - return Labels[label]; - } - public void AddLabel(string label, int value) - { - Labels[label] = value; - } - - public IEnumerable> LocalLabels() - { - foreach (KeyValuePair label in Labels) - { - yield return label; - } - } - } -} \ No newline at end of file diff --git a/ColorzCore/Parser/Definition.cs b/ColorzCore/Parser/Definition.cs deleted file mode 100644 index 2658b6f..0000000 --- a/ColorzCore/Parser/Definition.cs +++ /dev/null @@ -1,34 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.Lexer; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser -{ - public class Definition - { - private IList replacement; - - public Definition() - { - replacement = new List(); - } - - public Definition(IList defn) - { - replacement = defn; - } - - public IEnumerable ApplyDefinition(Token toReplace) - { - for(int i=0; i Definitions { get; } - public Dictionary> Raws { get; } - public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; - //public static readonly HashSet BuiltInMacros = new HashSet { "String", "AddToPool" }; - //TODO: Built in macros. - //public static readonly Dictionary BuiltInMacros; - public ImmutableStack GlobalScope { get; } - public int CurrentOffset { get { return currentOffset; } private set - { - if (value > 0x2000000) - { - if (validOffset) //Error only the first time. - { - Error(head == null ? new Location?() : head.Location, "Invalid offset: " + value.ToString("X")); - validOffset = false; - } - } - else - { - currentOffset = value; - validOffset = true; - offsetInitialized = true; - } - } - - } - public ImmutableStack Inclusion { get; set; } - - public Pool Pool { get; private set; } - - private readonly DirectiveHandler directiveHandler; - - private Stack> pastOffsets; // currentOffset, offsetInitialized - private IList> protectedRegions; - - public Log log; - - public bool IsIncluding { get - { - bool acc = true; - for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) - acc &= temp.Head; - return acc; - } } - private bool validOffset; - private bool offsetInitialized; // false until first ORG, used to warn about writing before first org - private int currentOffset; - private Token head; //TODO: Make this make sense - - public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) - { - GlobalScope = new ImmutableStack(new BaseClosure(this), ImmutableStack.Nil); - pastOffsets = new Stack>(); - protectedRegions = new List>(); - this.log = log; - Raws = raws; - CurrentOffset = 0; - validOffset = true; - offsetInitialized = false; - Macros = new MacroCollection(this); - Definitions = new Dictionary(); - Inclusion = ImmutableStack.Nil; - this.directiveHandler = directiveHandler; - - Pool = new Pool(); - } - - public bool IsReservedName(string name) - { - return Raws.ContainsKey(name.ToUpper()) || SpecialCodes.Contains(name.ToUpper()); - } - public bool IsValidDefinitionName(string name) - { - return !(Definitions.ContainsKey(name) || IsReservedName(name)); - } - public bool IsValidMacroName(string name, int paramNum) - { - return !(Macros.HasMacro(name, paramNum)) && !IsReservedName(name); - } - public bool IsValidLabelName(string name) - { - return true;//!IsReservedName(name); - //TODO? - } - public IList ParseAll(IEnumerable tokenStream) - { - //TODO: Make BlockNode or EAProgramNode? - //Note must be strict to get all information on the closure before evaluating terms. - IList myLines = new List(); - MergeableGenerator tokens = new MergeableGenerator(tokenStream); - tokens.MoveNext(); - while (!tokens.EOS) - { - if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) - { - Maybe retVal = ParseLine(tokens, GlobalScope); - retVal.IfJust( (ILineNode n) => myLines.Add(n)); - } - } - return myLines; - } - - private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack scopes) - { - Location start = tokens.Current.Location; - tokens.MoveNext(); - BlockNode temp = new BlockNode(); - Maybe x; - while (!tokens.EOS && tokens.Current.Type != TokenType.CLOSE_BRACE) - { - if (!(x = ParseLine(tokens, scopes)).IsNothing) - temp.Children.Add(x.FromJust); - } - if (!tokens.EOS) - tokens.MoveNext(); - else - Error(start, "Unmatched brace."); - return temp; - } - private Maybe ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) - { - while (ExpandIdentifier(tokens, scopes)) { } - head = tokens.Current; - tokens.MoveNext(); - //TODO: Replace with real raw information, and error if not valid. - IList parameters; - //TODO: Make intelligent to reject malformed parameters. - //TODO: Parse parameters after checking code validity. - if (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON) - { - parameters = ParseParamList(tokens, scopes); - } - else - { - parameters = new List(); - tokens.MoveNext(); - } - - string upperCodeIdentifier = head.Content.ToUpperInvariant(); - - if (SpecialCodes.Contains(upperCodeIdentifier)) - { - switch (upperCodeIdentifier) - { - case "ORG": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ORG: " + parameters.Count); - else - { - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => - { - if (temp > 0x2000000) - Error(parameters[0].MyLocation, "Tried to set offset to 0x" + temp.ToString("X")); - else - CurrentOffset = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to ORG."); } - ); - } - break; - case "PUSH": - if (parameters.Count != 0) - Error(head.Location, "Incorrect number of parameters in PUSH: " + parameters.Count); - else - pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); - break; - case "POP": - if (parameters.Count != 0) - Error(head.Location, "Incorrect number of parameters in POP: " + parameters.Count); - else if (pastOffsets.Count == 0) - Error(head.Location, "POP without matching PUSH."); - else { - Tuple tuple = pastOffsets.Pop(); - - CurrentOffset = tuple.Item1; - offsetInitialized = tuple.Item2; - } - break; - case "MESSAGE": - Message(head.Location, PrettyPrintParams(parameters)); - break; - case "WARNING": - Warning(head.Location, PrettyPrintParams(parameters)); - break; - case "ERROR": - Error(head.Location, PrettyPrintParams(parameters)); - break; - case "ASSERT": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ASSERT: " + parameters.Count); - else - { - - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => - { - atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => - { - if (temp < 0) - Error(parameters[0].MyLocation, "Assertion error: " + temp); - }); - }, - () => { Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); } - ); - } - break; - case "PROTECT": - if (parameters.Count == 1) - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => - { - protectedRegions.Add(new Tuple(temp, 4, head.Location)); - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); - else if (parameters.Count == 2) - { - int start = 0, end = 0; - bool errorOccurred = false; - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - (int temp) => - { - start = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - parameters[1].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - (int temp) => - { - end = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - if (!errorOccurred) - { - int length = end - start; - if (length > 0) - protectedRegions.Add(new Tuple(start, length, head.Location)); - else - Warning(head.Location, "Protected region not valid (end offset not after start offset). No region protected."); - } - } - else - Error(head.Location, "Incorrect number of parameters in PROTECT: " + parameters.Count); - break; - case "ALIGN": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ALIGN: " + parameters.Count); - else - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => - { - CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to ALIGN"); } - ); +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; +using ColorzCore.Preprocessor; +using ColorzCore.Preprocessor.Macros; +using ColorzCore.Raws; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +//TODO: Make errors less redundant (due to recursive nature, many paths will give several redundant errors). + +namespace ColorzCore.Parser +{ + public class EAParser + { + public Dictionary Definitions { get; } + public MacroCollection Macros { get; } + public DirectiveHandler DirectiveHandler { get; } + + public Dictionary> Raws { get; } + + public ImmutableStack Inclusion { get; set; } + + public Logger Logger { get; } + + public bool IsIncluding + { + get + { + bool acc = true; + + for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) + { + acc &= temp.Head; + } + + return acc; + } + } + + public delegate IAtomNode BindIdentifierFunc(Token identifierToken); + + public IParseConsumer ParseConsumer { get; } + + // TODO: IParseContextProvider or something like that? + public BindIdentifierFunc BindIdentifier { get; } + public StringProcessor StringProcessor { get; } + + public EAParser(Logger log, Dictionary> raws, IParseConsumer parseConsumer, BindIdentifierFunc bindIdentifier, StringProcessor stringProcessor) + { + Logger = log; + Raws = raws; + Macros = new MacroCollection(this); + Definitions = new Dictionary(); + Inclusion = ImmutableStack.Nil; + DirectiveHandler = new DirectiveHandler(); + ParseConsumer = parseConsumer; + BindIdentifier = bindIdentifier; + StringProcessor = stringProcessor; + } + + public bool IsReservedName(string name) + { + return Raws.ContainsKey(name.ToUpperInvariant()) || SpecialCodes.Contains(name.ToUpperInvariant()); + } + + public bool IsValidDefinitionName(string name) + { + return !(Definitions.ContainsKey(name) || IsReservedName(name)); + } + + public bool IsValidMacroName(string name, int paramNum) + { + return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); + } + + public void ParseAll(IEnumerable tokenStream) + { + MergeableGenerator tokens = new MergeableGenerator(tokenStream); + tokens.MoveNext(); + + while (!tokens.EOS) + { + if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) + { + ParseLine(tokens); + } + } + } + + public static readonly HashSet SpecialCodes = new HashSet() + { + "ORG", + "PUSH", + "POP", + "MESSAGE", + "WARNING", + "ERROR", + "ASSERT", + "PROTECT", + "ALIGN", + "FILL", + "STRING", + "BASE64", + // "SECTION", // TODO + // "DSECTION", // TODO + }; + + private void ParseStatement(MergeableGenerator tokens) + { + // NOTE: here previously lied en ExpandIdentifier loop + // though because this is only called from ParseLine after the corresponding check, this is not needed + + Token head = tokens.Current; + tokens.MoveNext(); + + switch (tokens.Current.Type) + { + case TokenType.COLON: + tokens.MoveNext(); + ParseConsumer.OnLabel(head.Location, head.Content); + return; + + case TokenType.ASSIGN: + tokens.MoveNext(); + ParseAssignment(head, head.Content, tokens); + return; + } + + // NOTE: those remarks are old ones from Colorz, idrk what they mean -Stan + // TODO: Replace with real raw information, and error if not valid. + // TODO: Make intelligent to reject malformed parameters. + // TODO: Parse parameters after checking code validity. + + IList parameters = tokens.Current.Type switch + { + TokenType.NEWLINE or TokenType.SEMICOLON => new List(), + _ => ParseParamList(tokens), + }; + + string upperCodeIdentifier = head.Content.ToUpperInvariant(); + + if (SpecialCodes.Contains(upperCodeIdentifier)) + { + switch (upperCodeIdentifier) + { + case "ORG": + ParseOrgStatement(head, parameters); + break; + + case "PUSH": + ParsePushStatement(head, parameters); + break; + + case "POP": + ParsePopStatement(head, parameters); break; - case "FILL": - if (parameters.Count > 2 || parameters.Count == 0) + + case "ASSERT": + ParseAssertStatement(head, parameters); + break; + + case "PROTECT": + ParseProtectStatement(head, parameters); + break; + + case "ALIGN": + ParseAlignStatement(head, parameters); + break; + + case "FILL": + ParseFillStatement(head, parameters); + break; + + case "MESSAGE": + ParseMessageStatement(head, parameters); + break; + + case "WARNING": + ParseWarningStatement(head, parameters); + break; + + case "ERROR": + ParseErrorStatement(head, parameters); + break; + + case "STRING": + ParseStringStatement(head, parameters); + break; + + case "BASE64": + ParseBase64Statement(head, parameters); + break; + } + } + else + { + ParseRawStatement(head, upperCodeIdentifier, tokens, parameters); + } + } + + private void ParseAssignment(Token head, string name, MergeableGenerator tokens) + { + IAtomNode? atom = this.ParseAtom(tokens); + + if (atom != null) + { + ParseConsumer.OnSymbolAssignment(head.Location, name, atom); + } + else + { + Logger.Error(head.Location, $"Couldn't define symbol `{name}`: exprected expression."); + } + } + + private void ParseRawStatement(Token head, string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) + { + if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) + { + // find the raw matching with parameters + foreach (Raw raw in raws) + { + if (raw.Fits(parameters)) + { + ParseConsumer.OnRawStatement(head.Location, raw, parameters); + return; + } + } + + if (raws.Count == 1) + { + Logger.Error(head.Location, $"Incorrect parameters in raw `{raws[0].ToPrettyString()}`"); + } + else + { + StringBuilder sb = new StringBuilder(); + + sb.Append($"Couldn't find suitable variant of raw `{upperCodeIdentifier}`."); + + for (int i = 0; i < raws.Count; i++) + { + sb.Append($"\nVariant {i + 1}: `{raws[i].ToPrettyString()}`"); + } + + Logger.Error(head.Location, sb.ToString()); + } + + IgnoreRestOfStatement(tokens); + } + else + { + Logger.Error(head.Location, $"Unrecognized statement code: {upperCodeIdentifier}"); + } + } + + private void ParseOrgStatement(Token head, IList parameters) + { + ParseStatementOneParam(head, "ORG", parameters, ParseConsumer.OnOrgStatement); + } + + private void ParsePushStatement(Token head, IList parameters) + { + if (parameters.Count == 0) + { + ParseConsumer.OnPushStatement(head.Location); + } + else + { + StatementExpectsParamCount(head, "PUSH", parameters, 0, 0); + } + } + + private void ParsePopStatement(Token head, IList parameters) + { + if (parameters.Count == 0) + { + ParseConsumer.OnPopStatement(head.Location); + } + else + { + StatementExpectsParamCount(head, "POP", parameters, 0, 0); + } + } + + private void ParseAssertStatement(Token head, IList parameters) + { + ParseStatementOneParam(head, "ASSERT", parameters, ParseConsumer.OnAssertStatement); + } + + // Helper method for printing errors + private void StatementExpectsAtom(string statementName, IParamNode param) + { + Logger.Error(param.MyLocation, + $"{statementName} expects an Atom (got {DiagnosticsHelpers.PrettyParamType(param.Type)})."); + } + + // Helper method for printing errors + private void StatementExpectsParamCount(Token head, string statementName, IList parameters, int min, int max) + { + if (min == max) + { + Logger.Error(head.Location, $"A {statementName} statement expects {min} parameters, got {parameters.Count}."); + } + else + { + Logger.Error(head.Location, $"A {statementName} statement expects {min} to {max} parameters, got {parameters.Count}."); + } + } + + private delegate void HandleStatementOne(Location location, IAtomNode node); + private delegate void HandleStatementTwo(Location location, IAtomNode firstNode, IAtomNode? optionalSecondNode); + + private void ParseStatementOneParam(Token head, string name, IList parameters, HandleStatementOne handler) + { + if (parameters.Count == 1) + { + if (parameters[0] is IAtomNode expr) + { + handler(head.Location, expr); + } + else + { + StatementExpectsAtom(name, parameters[0]); + } + } + else + { + StatementExpectsParamCount(head, name, parameters, 1, 1); + } + } + + private void ParseStatementTwoParam(Token head, string name, IList parameters, HandleStatementTwo handler) + { + if (parameters.Count == 1) + { + if (parameters[0] is IAtomNode firstNode) + { + handler(head.Location, firstNode, null); + } + else + { + StatementExpectsAtom(name, parameters[0]); + } + } + else if (parameters.Count == 2) + { + if (parameters[0] is IAtomNode firstNode) + { + if (parameters[1] is IAtomNode secondNode) + { + handler(head.Location, firstNode, secondNode); + } + else + { + StatementExpectsAtom(name, parameters[1]); + } + } + else + { + StatementExpectsAtom(name, parameters[0]); + } + } + else + { + StatementExpectsParamCount(head, name, parameters, 1, 2); + } + } + + private void ParseProtectStatement(Token head, IList parameters) + { + ParseStatementTwoParam(head, "PROTECT", parameters, ParseConsumer.OnProtectStatement); + } + + private void ParseAlignStatement(Token head, IList parameters) + { + ParseStatementTwoParam(head, "ALIGN", parameters, ParseConsumer.OnAlignStatement); + } + + private void ParseFillStatement(Token head, IList parameters) + { + ParseStatementTwoParam(head, "FILL", parameters, ParseConsumer.OnFillStatement); + } + + private void ParseMessageStatement(Token head, IList parameters) + { + Logger.Message(head.Location, PrettyPrintParamsForMessage(parameters)); + } + + private void ParseWarningStatement(Token head, IList parameters) + { + Logger.Warning(head.Location, PrettyPrintParamsForMessage(parameters)); + } + + private void ParseErrorStatement(Token head, IList parameters) + { + Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters)); + } + + private void ParseStringStatement(Token head, IList parameters) + { + void HandleStringStatement(Token head, StringNode node, string? encodingName) + { + string formattedString = StringProcessor.ExpandUserFormatString( + node.MyLocation, this, node.Value); + + byte[] encodedString = StringProcessor.EncodeString( + head.Location, formattedString, encodingName); + + if (encodedString.Length != 0) + { + ParseConsumer.OnData(head.Location, encodedString); + } + } + + // NOTE: this is copy-pasted from ParseStatementTwoParam but adjusted for string param + + if (parameters.Count == 1) + { + if (parameters[0] is StringNode firstNode) + { + HandleStringStatement(head, firstNode, null); + } + else + { + Logger.Error(parameters[0].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[0].Type)})."); + } + } + else if (parameters.Count == 2) + { + if (parameters[0] is StringNode firstNode) + { + if (parameters[1] is StringNode secondNode) + { + HandleStringStatement(head, firstNode, secondNode.Value); + } + else + { + Logger.Error(parameters[1].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[1].Type)})."); + } + } + else + { + Logger.Error(parameters[0].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[0].Type)})."); + } + } + else + { + Logger.Error(head.Location, + $"A STRING statement expects 1 to 2 parameters, got {parameters.Count}."); + } + } + + private void ParseBase64Statement(Token head, IList parameters) + { + using MemoryStream memoryStream = new MemoryStream(); + + foreach (IParamNode parameter in parameters) + { + if (parameter is StringNode node) + { + try + { + byte[] base64Bytes = Convert.FromBase64String(node.Value); + memoryStream.Write(base64Bytes, 0, base64Bytes.Length); + } + catch (FormatException e) + { + Logger.Error(node.MyLocation, $"Failed to parse Base64 string: {e.Message}"); + return; + } + } + else + { + Logger.Error(head.Location, $"expects a String (got {DiagnosticsHelpers.PrettyParamType(parameter.Type)})."); + } + } + + byte[] bytes = memoryStream.ToArray(); + ParseConsumer.OnData(head.Location, bytes); + } + + public IList> ParseMacroParamList(MergeableGenerator tokens) + { + IList> parameters = new List>(); + int parenNestings = 0; + + // HACK: this allows macro([1, 2, 3]) from expanding into a single parameter + int bracketBalance = 0; + + do + { + tokens.MoveNext(); + List currentParam = new List(); + while ( + !(parenNestings == 0 + && (tokens.Current.Type == TokenType.CLOSE_PAREN || (bracketBalance == 0 && tokens.Current.Type == TokenType.COMMA))) + && tokens.Current.Type != TokenType.NEWLINE) + { + switch (tokens.Current.Type) + { + case TokenType.CLOSE_PAREN: + parenNestings--; + break; + case TokenType.OPEN_PAREN: + parenNestings++; + break; + case TokenType.OPEN_BRACKET: + bracketBalance++; + break; + case TokenType.CLOSE_BRACKET: + bracketBalance--; + break; + } + + currentParam.Add(tokens.Current); + tokens.MoveNext(); + } + + parameters.Add(currentParam); + } + while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); + + if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) + { + Logger.Error(tokens.Current.Location, "Unmatched open parenthesis."); + } + else + { + tokens.MoveNext(); + } + + return parameters; + } + + private IList ParseParamList(MergeableGenerator tokens) + { + IList paramList = new List(); + + while (!IsTokenAlwaysPastEndOfStatement(tokens.Current) && !tokens.EOS) + { + Token localHead = tokens.Current; + ParseParam(tokens).IfJust( + n => paramList.Add(n), + () => Logger.Error(localHead.Location, "Expected parameter.")); + } + + if (tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + } + + return paramList; + } + + private static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); + + public IList ParsePreprocParamList(MergeableGenerator tokens) + { + static bool IsValidIdentifier(string value) + { + return idRegex.IsMatch(value); + } + + IList temp = ParseParamList(tokens); + + for (int i = 0; i < temp.Count; i++) + { + if (temp[i] is StringNode stringNode && IsValidIdentifier(stringNode.Value)) + { + // TODO: what is this for? can we omit it? + temp[i] = BindIdentifier(stringNode.SourceToken); + } + } + + return temp; + } + + public IParamNode? ParseParam(MergeableGenerator tokens) + { + Token localHead = tokens.Current; + switch (localHead.Type) + { + case TokenType.OPEN_BRACKET: + return new ListNode(localHead.Location, ParseList(tokens)); + case TokenType.STRING: + tokens.MoveNext(); + return new StringNode(localHead); + case TokenType.MAYBE_MACRO: + //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. + if (ExpandIdentifier(tokens, true)) + { + return ParseParam(tokens); + } + else + { + tokens.MoveNext(); + IList> param = ParseMacroParamList(tokens); + //TODO: Smart errors if trying to redefine a macro with the same num of params. + return new MacroInvocationNode(this, localHead, param); + } + case TokenType.IDENTIFIER: + if (ExpandIdentifier(tokens, true)) + { + return ParseParam(tokens); + } + else + { + switch (localHead.Content.ToUpperInvariant()) { - Error(head.Location, "Incorrect number of parameters in FILL: " + parameters.Count); - } - else - { - // FILL [value] - - int amount = 0; - int value = 0; - - if (parameters.Count == 2) - { - // param 2 (if given) is fill value - - parameters[1].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int val) => { value = val; }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); + case "__FILE__": + tokens.MoveNext(); + return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); + + default: + return this.ParseAtom(tokens); + } + } + + default: + return this.ParseAtom(tokens); + } + } + + private IList ParseList(MergeableGenerator tokens) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + IList atoms = new List(); + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) + { + IAtomNode? res = this.ParseAtom(tokens); + res.IfJust( + n => atoms.Add(n), + () => Logger.Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); + if (tokens.Current.Type == TokenType.COMMA) + { + tokens.MoveNext(); + } + } + if (tokens.Current.Type == TokenType.CLOSE_BRACKET) + { + tokens.MoveNext(); + } + else + { + Logger.Error(localHead.Location, "Unmatched open bracket."); + } + + return atoms; + } + + public void ParseLine(MergeableGenerator tokens) + { + if (IsIncluding) + { + if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + return; + } + + Token head = tokens.Current; + switch (head.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + if (ExpandIdentifier(tokens)) + { + // NOTE: we check here if we didn't end up with something that can't be a statement + + switch (tokens.Current.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.OPEN_BRACE: + case TokenType.PREPROCESSOR_DIRECTIVE: + // recursion! + ParseLine(tokens); + return; + + default: + // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' + Logger.Error(head.Location, $"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); + IgnoreRestOfStatement(tokens); + break; } - - // param 1 is amount of bytes to fill - parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int val) => { amount = val; }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); - - var data = new byte[amount]; - - for (int i = 0; i < amount; ++i) - data[i] = (byte) value; - - var node = new DataNode(CurrentOffset, data); - - CheckDataWrite(amount); - CurrentOffset += amount; - - return new Just(node); - } - - break; - } - return new Nothing(); - } - else if (Raws.ContainsKey(upperCodeIdentifier)) - { - //TODO: Check for matches. Currently should type error. - foreach(Raw r in Raws[upperCodeIdentifier]) - { - if (r.Fits(parameters)) - { - if ((CurrentOffset % r.Alignment) != 0) - { - Error(head.Location, string.Format("Bad code alignment (offset: {0:X8})", CurrentOffset)); - - } - StatementNode temp = new RawNode(r, head, CurrentOffset, parameters); - - CheckDataWrite(temp.Size); - CurrentOffset += temp.Size; //TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - - return new Just(temp); - } - } - //TODO: Better error message (a la EA's ATOM ATOM [ATOM,ATOM]) - Error(head.Location, "Incorrect parameters in raw " + head.Content + '.'); - IgnoreRestOfStatement(tokens); - return new Nothing(); - } - else //TODO: Move outside of this else. - { - Error(head.Location, "Unrecognized code: " + head.Content); - return new Nothing(); - } - } - - public IList> ParseMacroParamList(MergeableGenerator tokens) - { - IList> parameters = new List>(); - int parenNestings = 0; - do - { - tokens.MoveNext(); - List currentParam = new List(); - while ( - !(parenNestings == 0 && (tokens.Current.Type == TokenType.CLOSE_PAREN || tokens.Current.Type == TokenType.COMMA)) - && tokens.Current.Type != TokenType.NEWLINE) - { - if (tokens.Current.Type == TokenType.CLOSE_PAREN) - parenNestings--; - else if (tokens.Current.Type == TokenType.OPEN_PAREN) - parenNestings++; - currentParam.Add(tokens.Current); - tokens.MoveNext(); - } - parameters.Add(currentParam); - } while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); - if(tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) - { - Error(tokens.Current.Location, "Unmatched open parenthesis."); - } - else - { - tokens.MoveNext(); - } - return parameters; - } - - private IList ParseParamList(MergeableGenerator tokens, ImmutableStack scopes, bool expandFirstDef = true) - { - IList paramList = new List(); - bool first = true; - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && !tokens.EOS) - { - Token head = tokens.Current; - ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( - (IParamNode n) => paramList.Add(n), - () => Error(head.Location, "Expected parameter.")); - first = false; - } - if (tokens.Current.Type == TokenType.SEMICOLON) - tokens.MoveNext(); - return paramList; - } - - private IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes) - { - IList temp = ParseParamList(tokens, scopes, false); - for(int i=0; i ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) - { - Token head = tokens.Current; - switch (tokens.Current.Type) - { - case TokenType.OPEN_BRACKET: - return new Just(new ListNode(head.Location, ParseList(tokens, scopes)).Simplify()); - case TokenType.STRING: - tokens.MoveNext(); - return new Just(new StringNode(head)); - case TokenType.MAYBE_MACRO: - //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. - if (expandDefs && ExpandIdentifier(tokens, scopes)) - { - return ParseParam(tokens, scopes); - } - else - { - tokens.MoveNext(); - IList> param = ParseMacroParamList(tokens); - //TODO: Smart errors if trying to redefine a macro with the same num of params. - return new Just(new MacroInvocationNode(this, head, param, scopes)); - } - case TokenType.IDENTIFIER: - if (expandDefs && Definitions.ContainsKey(head.Content) && ExpandIdentifier(tokens, scopes)) - return ParseParam(tokens, scopes, expandDefs); - else - return ParseAtom(tokens,scopes,expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); - default: - return ParseAtom(tokens, scopes, expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); - } - } - - private static readonly Dictionary precedences = new Dictionary { - { TokenType.MUL_OP , 3 }, - { TokenType.DIV_OP , 3 }, - { TokenType.ADD_OP , 4 }, - { TokenType.SUB_OP , 4 }, - { TokenType.LSHIFT_OP , 5 }, - { TokenType.RSHIFT_OP , 5 }, - { TokenType.SIGNED_RSHIFT_OP , 5 }, - { TokenType.AND_OP , 8 }, - { TokenType.XOR_OP , 9 }, - { TokenType.OR_OP , 10 }, - { TokenType.MOD_OP , 3 } - }; - - - - private Maybe ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) - { - //Use Shift Reduce Parsing - Token head = tokens.Current; - Stack> grammarSymbols = new Stack>(); - bool ended = false; - while (!ended) - { - bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; - Token lookAhead = tokens.Current; - - if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. - { - //Verify next symbol to be an operator. - switch (lookAhead.Type) - { - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.SUB_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - if (precedences.ContainsKey(lookAhead.Type)) - { - Reduce(grammarSymbols, precedences[lookAhead.Type]); - } - shift = true; - break; - default: - ended = true; - break; - } - } - else if (!ended) //Is just an operator. Error if two operators in a row. - { - //Error if two operators in a row. - switch (lookAhead.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - case TokenType.NUMBER: - shift = true; - break; - case TokenType.OPEN_PAREN: - { - tokens.MoveNext(); - Maybe interior = ParseAtom(tokens, scopes); - if (tokens.Current.Type != TokenType.CLOSE_PAREN) - { - Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); - return new Nothing(); - } - else if (interior.IsNothing) - { - Error(lookAhead.Location, "Expected expression inside paretheses. "); - return new Nothing(); - } - else - { - grammarSymbols.Push(new Left(interior.FromJust)); - tokens.MoveNext(); - break; - } - } - case TokenType.SUB_OP: - { - //Assume unary negation. - tokens.MoveNext(); - Maybe interior = ParseAtom(tokens, scopes); - if(interior.IsNothing) - { - Error(lookAhead.Location, "Expected expression after negation. "); - return new Nothing(); - } - grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior.FromJust))); - break; - } - case TokenType.COMMA: - Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); - IgnoreRestOfStatement(tokens); - return new Nothing(); - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - default: - Error(lookAhead.Location, "Expected identifier or literal, got " + lookAhead.Type + ": " + lookAhead.Content + '.'); - IgnoreRestOfStatement(tokens); - return new Nothing(); - } - } - - if (shift) - { - if (lookAhead.Type == TokenType.IDENTIFIER) - { - if (expandDefs && ExpandIdentifier(tokens, scopes)) - continue; - if (lookAhead.Content.ToUpper() == "CURRENTOFFSET") - grammarSymbols.Push(new Left(new NumberNode(lookAhead, CurrentOffset))); - else - grammarSymbols.Push(new Left(new IdentifierNode(lookAhead, scopes))); - } - else if (lookAhead.Type == TokenType.MAYBE_MACRO) - { - ExpandIdentifier(tokens, scopes); - continue; - } - else if (lookAhead.Type == TokenType.NUMBER) - { - grammarSymbols.Push(new Left(new NumberNode(lookAhead))); - } - else if (lookAhead.Type == TokenType.ERROR) - { - Error(lookAhead.Location, System.String.Format("Unexpected token: {0}", lookAhead.Content)); - tokens.MoveNext(); - return new Nothing(); - } - else - { - grammarSymbols.Push(new Right(lookAhead)); - } - tokens.MoveNext(); - continue; - } - } - while (grammarSymbols.Count > 1) - { - Reduce(grammarSymbols, 11); - } - if (grammarSymbols.Peek().IsRight) - { - Error(grammarSymbols.Peek().GetRight.Location, "Unexpected token: " + grammarSymbols.Peek().GetRight.Type); - } - return new Just(grammarSymbols.Peek().GetLeft); - } - - /*** - * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd - * the precedences of the IAtomNodes is increasing. - * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. - * - */ - private void Reduce(Stack> grammarSymbols, int targetPrecedence) - { - while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) - { - //These shouldn't error... - IAtomNode r = grammarSymbols.Pop().GetLeft; - - if(precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) - { - grammarSymbols.Push(new Left(r)); - break; - } - else - { - Token op = grammarSymbols.Pop().GetRight; - IAtomNode l = grammarSymbols.Pop().GetLeft; - - grammarSymbols.Push(new Left(new OperatorNode(l, op, r, l.Precedence))); - } - } - } - - private int GetLowestPrecedence(Stack> grammarSymbols) - { - int minPrec = 11; //TODO: Note that this is the largest possible value. - foreach (Either e in grammarSymbols) - e.Case((IAtomNode n) => { minPrec = Math.Min(minPrec, n.Precedence); }, - (Token t) => { minPrec = Math.Min(minPrec, precedences[t.Type]); }); - return minPrec; - } - - private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) - { - Token head = tokens.Current; - tokens.MoveNext(); - IList atoms = new List(); - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) - { - Maybe res = ParseAtom(tokens, scopes); - res.IfJust( - (IAtomNode n) => atoms.Add(n), - () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); - if (tokens.Current.Type == TokenType.COMMA) - tokens.MoveNext(); - } - if (tokens.Current.Type == TokenType.CLOSE_BRACKET) - tokens.MoveNext(); - else - Error(head.Location, "Unmatched open bracket."); - return atoms; - } - - public Maybe ParseLine(MergeableGenerator tokens, ImmutableStack scopes) - { - if (IsIncluding) - { - if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) - { - tokens.MoveNext(); - return new Nothing(); - } - head = tokens.Current; - switch (head.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - if (ExpandIdentifier(tokens, scopes)) - { - return ParseLine(tokens, scopes); - } - else - { - tokens.MoveNext(); - if (tokens.Current.Type == TokenType.COLON) - { - tokens.MoveNext(); - if (scopes.Head.HasLocalLabel(head.Content)) - { - Warning(head.Location, "Label already in scope, ignoring: " + head.Content);//replacing: " + head.Content); - } - else if(!IsValidLabelName(head.Content)) - { - Error(head.Location, "Invalid label name " + head.Content + '.'); - } - else - { - scopes.Head.AddLabel(head.Content, CurrentOffset); - } - - return new Nothing(); - } - else - { - tokens.PutBack(head); - return ParseStatement(tokens, scopes); - } - } - case TokenType.OPEN_BRACE: - return new Just(ParseBlock(tokens, new ImmutableStack(new Closure(), scopes))); - case TokenType.PREPROCESSOR_DIRECTIVE: - return ParsePreprocessor(tokens, scopes); - case TokenType.OPEN_BRACKET: - Error(head.Location, "Unexpected list literal."); - IgnoreRestOfLine(tokens); - break; - case TokenType.NUMBER: - case TokenType.OPEN_PAREN: - Error(head.Location, "Unexpected mathematical expression."); - IgnoreRestOfLine(tokens); - break; - default: - tokens.MoveNext(); - Error(head.Location, System.String.Format("Unexpected token: {0}: {1}", head.Type, head.Content)); - IgnoreRestOfLine(tokens); - break; - } - return new Nothing(); - } - else - { - bool hasNext = true; - while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) ; - if (hasNext) - { - return ParsePreprocessor(tokens, scopes); - } - else - { - Error(null, System.String.Format("Missing {0} endif(s).", Inclusion.Count)); - return new Nothing(); - } - } - } - - private Maybe ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) - { - head = tokens.Current; - tokens.MoveNext(); - //Note: Not a ParseParamList because no commas. - IList paramList = ParsePreprocParamList(tokens, scopes); - Maybe retVal = directiveHandler.HandleDirective(this, head, paramList, tokens); - if (!retVal.IsNothing) - { - CheckDataWrite(retVal.FromJust.Size); - CurrentOffset += retVal.FromJust.Size; - } - return retVal; - } - - /*** - * Precondition: tokens.Current.Type == TokenType.IDENTIFIER || MAYBE_MACRO - * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) - * Returns: true iff tokens was actually expanded. - */ - public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes) - { - bool ret = false; - //Macros and Definitions. - if (tokens.Current.Type == TokenType.MAYBE_MACRO && Macros.ContainsName(tokens.Current.Content)) - { - Token head = tokens.Current; - tokens.MoveNext(); - IList> parameters = ParseMacroParamList(tokens); - if (Macros.HasMacro(head.Content, parameters.Count)) - { - tokens.PrependEnumerator(Macros.GetMacro(head.Content, parameters.Count).ApplyMacro(head, parameters, scopes).GetEnumerator()); - } - else - { - Error(head.Location, System.String.Format("No overload of {0} with {1} parameters.", head.Content, parameters.Count)); - } - return true; - } - else if(tokens.Current.Type == TokenType.MAYBE_MACRO) - { - Token head = tokens.Current; - tokens.MoveNext(); - tokens.PutBack(new Token(TokenType.IDENTIFIER, head.Location, head.Content)); - return true; - } - else if (Definitions.ContainsKey(tokens.Current.Content)) - { - Token head = tokens.Current; - tokens.MoveNext(); - tokens.PrependEnumerator(Definitions[head.Content].ApplyDefinition(head).GetEnumerator()); - return true; - } - - return ret; - } - - public void Message(Location? loc, string message) - { - log.Message(Log.MsgKind.MESSAGE, loc, message); - } - - public void Warning(Location? loc, string message) - { - log.Message(Log.MsgKind.WARNING, loc, message); - } - - public void Error(Location? loc, string message) - { - log.Message(Log.MsgKind.ERROR, loc, message); - } - - private void IgnoreRestOfStatement(MergeableGenerator tokens) - { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) ; - if (tokens.Current.Type == TokenType.SEMICOLON) tokens.MoveNext(); - } - - private void IgnoreRestOfLine(MergeableGenerator tokens) - { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) ; - } - - public void Clear() - { - Macros.Clear(); - Definitions.Clear(); - Raws.Clear(); - Inclusion = ImmutableStack.Nil; - CurrentOffset = 0; - pastOffsets.Clear(); - } - - private string PrettyPrintParams(IList parameters) - { - StringBuilder sb = new StringBuilder(); - foreach(IParamNode parameter in parameters) - { - sb.Append(parameter.PrettyPrint()); - sb.Append(' '); - } - return sb.ToString(); - } - - // Return value: Location where protection occurred. Nothing if location was not protected. - private Maybe IsProtected(int offset, int length) - { - foreach (Tuple protectedRegion in protectedRegions) - { - //They intersect if the last offset in the given region is after the start of this one - //and the first offset in the given region is before the last of this one - if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) - return new Just(protectedRegion.Item3); - } - return new Nothing(); - } - - private void CheckDataWrite(int length) - { - // TODO: maybe make this warning optional? - if (!offsetInitialized) - { - Warning(head.Location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); - offsetInitialized = false; // only warn once - } - - // TODO (maybe?): save Location of PROTECT statement, for better diagnosis - // We would then print something like "Trying to write data to area protected at " - - Maybe prot = IsProtected(CurrentOffset, length); - if (!prot.IsNothing) - { - Location l = prot.FromJust; - Error(head.Location, System.String.Format("Trying to write data to area protected in file {0} at line {1}, column {2}.", Path.GetFileName(l.file), l.lineNum, l.colNum)); - } - } - } -} + + return; + } + else + { + ParseStatement(tokens); + } + + break; + + case TokenType.OPEN_BRACE: + tokens.MoveNext(); + ParseConsumer.OnOpenScope(head.Location); + break; + + case TokenType.CLOSE_BRACE: + tokens.MoveNext(); + ParseConsumer.OnCloseScope(head.Location); + break; + + case TokenType.PREPROCESSOR_DIRECTIVE: + ParsePreprocessor(tokens); + break; + + case TokenType.OPEN_BRACKET: + Logger.Error(head.Location, "Unexpected list literal."); + IgnoreRestOfStatement(tokens); + break; + + case TokenType.NUMBER: + case TokenType.OPEN_PAREN: + Logger.Error(head.Location, "Unexpected mathematical expression."); + IgnoreRestOfStatement(tokens); + break; + + default: + tokens.MoveNext(); + + if (string.IsNullOrEmpty(head.Content)) + { + Logger.Error(head.Location, $"Unexpected token: {head.Type}."); + } + else + { + Logger.Error(head.Location, $"Unexpected token: {head.Type}: {head.Content}."); + } + + IgnoreRestOfStatement(tokens); + break; + } + } + else + { + bool hasNext = true; + + while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) + { + } + + if (hasNext) + { + ParsePreprocessor(tokens); + } + else + { + Logger.Error(null, $"Missing {Inclusion.Count} endif(s)."); + } + } + } + + private void ParsePreprocessor(MergeableGenerator tokens) + { + Token head = tokens.Current; + tokens.MoveNext(); + DirectiveHandler.HandleDirective(this, head, tokens); + } + + /*** + * Precondition: tokens.Current.Type == TokenType.IDENTIFIER || MAYBE_MACRO + * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) + * Returns: true iff tokens was actually expanded. + */ + public bool ExpandIdentifier(MergeableGenerator tokens, bool insideExpression = false) + { + // function-like macros + if (tokens.Current.Type == TokenType.MAYBE_MACRO) + { + if (Macros.ContainsName(tokens.Current.Content)) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + IList> parameters = ParseMacroParamList(tokens); + + if (Macros.TryGetMacro(localHead.Content, parameters.Count, out IMacro? macro)) + { + /* macro is 100% not null here, but because we can't use NotNullWhen on TryGetMacro, + * since the attribute is unavailable in .NET Framework (which we still target), + * the compiler will still diagnose a nullable dereference if we don't use '!' also */ + + ApplyMacroExpansion(tokens, macro!.ApplyMacro(localHead, parameters), insideExpression); + } + else + { + Logger.Error(localHead.Location, $"No overload of {localHead.Content} with {parameters.Count} parameters."); + } + + return true; + } + else + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); + return true; + } + } + + // object-like macros (aka "Definitions") + if (Definitions.TryGetValue(tokens.Current.Content, out Definition? definition) && !definition.NonProductive) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + ApplyMacroExpansion(tokens, definition.ApplyDefinition(localHead), insideExpression); + return true; + } + + return false; + } + + private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedTokens, bool insideExpression = false) + { + if (insideExpression && EAOptions.IsWarningEnabled(EAOptions.Warnings.UnguardedExpressionMacros)) + { + // here we check for any operator that isn't enclosed in parenthesises + + IList expandedList = expandedTokens.ToList(); + + DiagnosticsHelpers.VisitUnguardedOperators(expandedList, + token => Logger.Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); + + tokens.PrependEnumerator(expandedList.GetEnumerator()); + } + else + { + tokens.PrependEnumerator(expandedTokens.GetEnumerator()); + } + } + + public void IgnoreRestOfStatement(MergeableGenerator tokens) + { + while (!IsTokenAlwaysPastEndOfStatement(tokens.Current) && tokens.MoveNext()) + { + } + + if (tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + } + } + + public void IgnoreRestOfLine(MergeableGenerator tokens) + { + while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) + { + } + } + + /// + /// Consumes incoming tokens util end of line. + /// + /// token stream + /// If non-null, will expand any macros as they are encountered using this scope + /// The resulting list of tokens + public IList GetRestOfLine(MergeableGenerator tokens) + { + IList result = new List(); + + while (tokens.Current.Type != TokenType.NEWLINE) + { + if (!ExpandIdentifier(tokens)) + { + result.Add(tokens.Current); + tokens.MoveNext(); + } + } + + return result; + } + + private static bool IsTokenAlwaysPastEndOfStatement(Token token) => token.Type switch + { + TokenType.NEWLINE => true, + TokenType.SEMICOLON => true, + TokenType.OPEN_BRACE => true, + TokenType.CLOSE_BRACE => true, + _ => false, + }; + + private string PrettyPrintParamsForMessage(IList parameters) + { + return string.Join(" ", parameters.Select(parameter => parameter switch + { + StringNode node => StringProcessor.ExpandUserFormatString(parameter.MyLocation, this, node.Value), + _ => parameter.PrettyPrint(), + })); + } + } +} diff --git a/ColorzCore/Parser/IParseConsumer.cs b/ColorzCore/Parser/IParseConsumer.cs new file mode 100644 index 0000000..221f2ac --- /dev/null +++ b/ColorzCore/Parser/IParseConsumer.cs @@ -0,0 +1,24 @@ +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; +using ColorzCore.Raws; + +namespace ColorzCore.Parser +{ + public interface IParseConsumer + { + void OnOpenScope(Location location); + void OnCloseScope(Location location); + void OnRawStatement(Location location, Raw raw, IList parameters); + void OnOrgStatement(Location location, IAtomNode offsetNode); + void OnPushStatement(Location location); + void OnPopStatement(Location location); + void OnAssertStatement(Location location, IAtomNode node); + void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom); + void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode); + void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode); + void OnSymbolAssignment(Location location, string name, IAtomNode atom); + void OnLabel(Location location, string name); + void OnData(Location location, byte[] data); + } +} diff --git a/ColorzCore/Parser/Macros/Macro.cs b/ColorzCore/Parser/Macros/Macro.cs deleted file mode 100644 index 78581ae..0000000 --- a/ColorzCore/Parser/Macros/Macro.cs +++ /dev/null @@ -1,48 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.Lexer; -using ColorzCore.Parser.AST; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.Macros -{ - class Macro : IMacro - { - Dictionary idToParamNum; - IList body; - - public Macro(IList parameters, IList body) - { - idToParamNum = new Dictionary(); - for(int i=0; i ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) - { - foreach(Token t in body) - { - if(t.Type == TokenType.IDENTIFIER && idToParamNum.ContainsKey(t.Content)) - { - foreach(Token t2 in parameters[idToParamNum[t.Content]]) - { - yield return t2; - } - } - else - { - yield return new Token(t.Type, head.Location, t.Content); - } - } - } - } -} diff --git a/ColorzCore/Parser/Macros/String.cs b/ColorzCore/Parser/Macros/String.cs deleted file mode 100644 index 4366673..0000000 --- a/ColorzCore/Parser/Macros/String.cs +++ /dev/null @@ -1,25 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.Lexer; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.Macros -{ - class String : BuiltInMacro - { - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) - { - yield return new Token(TokenType.IDENTIFIER, head.Location, "BYTE"); - foreach (byte num in Encoding.ASCII.GetBytes(parameters[0][0].Content.ToCharArray())) //TODO: Errors if not adherent? - yield return new Token(TokenType.NUMBER, head.Location, num.ToString()); - } - - public override bool ValidNumParams(int num) - { - return num == 1; - } - } -} diff --git a/ColorzCore/Parser/ParseConsumerChain.cs b/ColorzCore/Parser/ParseConsumerChain.cs new file mode 100644 index 0000000..cd8879d --- /dev/null +++ b/ColorzCore/Parser/ParseConsumerChain.cs @@ -0,0 +1,76 @@ + +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; +using ColorzCore.Raws; + +namespace ColorzCore.Parser +{ + public class ParseConsumerChain : List, IParseConsumer + { + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) + { + ForEach(pc => pc.OnAlignStatement(location, alignNode, offsetNode)); + } + + public void OnAssertStatement(Location location, IAtomNode node) + { + ForEach(pc => pc.OnAssertStatement(location, node)); + } + + public void OnCloseScope(Location location) + { + ForEach(pc => pc.OnCloseScope(location)); + } + + public void OnData(Location location, byte[] data) + { + ForEach(pc => pc.OnData(location, data)); + } + + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) + { + ForEach(pc => pc.OnFillStatement(location, amountNode, valueNode)); + } + + public void OnLabel(Location location, string name) + { + ForEach(pc => pc.OnLabel(location, name)); + } + + public void OnOpenScope(Location location) + { + ForEach(pc => pc.OnOpenScope(location)); + } + + public void OnOrgStatement(Location location, IAtomNode offsetNode) + { + ForEach(pc => pc.OnOrgStatement(location, offsetNode)); + } + + public void OnPopStatement(Location location) + { + ForEach(pc => pc.OnPopStatement(location)); + } + + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) + { + ForEach(pc => pc.OnProtectStatement(location, beginAtom, endAtom)); + } + + public void OnPushStatement(Location location) + { + ForEach(pc => pc.OnPushStatement(location)); + } + + public void OnRawStatement(Location location, Raw raw, IList parameters) + { + ForEach(pc => pc.OnRawStatement(location, raw, parameters)); + } + + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) + { + ForEach(pc => pc.OnSymbolAssignment(location, name, atom)); + } + } +} \ No newline at end of file diff --git a/ColorzCore/Parser/Pool.cs b/ColorzCore/Parser/Pool.cs index b8b73c5..f02ece1 100644 --- a/ColorzCore/Parser/Pool.cs +++ b/ColorzCore/Parser/Pool.cs @@ -5,16 +5,14 @@ namespace ColorzCore.Parser { - class Pool + public class Pool { public struct PooledLine { - public ImmutableStack Scope { get; private set; } public List Tokens { get; private set; } - public PooledLine(ImmutableStack scope, List tokens) + public PooledLine(List tokens) { - Scope = scope; Tokens = tokens; } } @@ -34,7 +32,7 @@ public Pool() public string MakePoolLabelName() { // The presence of $ in the label name guarantees that it can't be a user label - return string.Format("{0}{1}", pooledLabelPrefix, poolLabelCounter++); + return $"{pooledLabelPrefix}{poolLabelCounter++}"; } } } diff --git a/ColorzCore/Preprocessor/Definition.cs b/ColorzCore/Preprocessor/Definition.cs new file mode 100644 index 0000000..b65cdc2 --- /dev/null +++ b/ColorzCore/Preprocessor/Definition.cs @@ -0,0 +1,47 @@ +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ColorzCore.Preprocessor +{ + public class Definition + { + private readonly IList? replacement; + + /// + /// A non-productive definition is a definition that doesn't participate in macro expansion. + /// (it is still visible by ifdef and other such constructs). + /// + public bool NonProductive => replacement == null; + + public Definition() + { + replacement = null; + } + + public Definition(Token token) + { + replacement = new List { token }; + } + + public Definition(IList defn) + { + replacement = defn; + } + + public IEnumerable ApplyDefinition(Token token) + { + // assumes !NonProductive + + IList replacement = this.replacement!; + MacroLocation macroLocation = new MacroLocation(token.Content, token.Location); + + for (int i = 0; i < replacement.Count; i++) + { + yield return replacement[i].MacroClone(macroLocation); + } + } + } +} diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 3cf8ff2..cbbcb3c 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -1,64 +1,50 @@ using ColorzCore.DataTypes; -using ColorzCore.IO; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; using ColorzCore.Preprocessor.Directives; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Preprocessor { - class DirectiveHandler + public class DirectiveHandler { - private Dictionary directives; + // TODO: do we need this class? Could we not just have this part of EAParser? - public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher toolSearcher) - { - directives = new Dictionary + public Dictionary Directives { get; } + + public DirectiveHandler() + { + Directives = new Dictionary { - { "include", new IncludeDirective { FileSearcher = includeSearcher } }, - { "incbin", new IncludeBinaryDirective { FileSearcher = includeSearcher } }, - { "incext", new IncludeExternalDirective { FileSearcher = toolSearcher } }, - { "inctext", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, - { "inctevent", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, - { "ifdef", new IfDefinedDirective() }, - { "ifndef", new IfNotDefinedDirective() }, + { "ifdef", new IfDefinedDirective(false) }, + { "ifndef", new IfDefinedDirective(true) }, + { "if", new IfDirective() }, { "else", new ElseDirective() }, { "endif", new EndIfDirective() }, { "define", new DefineDirective() }, - { "pool", new PoolDirective() }, { "undef", new UndefineDirective() }, - }; + }; } - public Maybe HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) + public void HandleDirective(EAParser p, Token directive, MergeableGenerator tokens) { string directiveName = directive.Content.Substring(1); - if (directives.TryGetValue(directiveName, out IDirective toExec)) + if (Directives.TryGetValue(directiveName, out IDirective? toExec)) { if (!toExec.RequireInclusion || p.IsIncluding) { - if (toExec.MinParams <= parameters.Count && (!toExec.MaxParams.HasValue || parameters.Count <= toExec.MaxParams)) - { - return toExec.Execute(p, directive, parameters, tokens); - } - else - { - p.Error(directive.Location, "Invalid number of parameters (" + parameters.Count + ") to directive " + directiveName + "."); - } + toExec.Execute(p, directive, tokens); } } else { - p.Error(directive.Location, "Directive not recognized: " + directiveName); + p.Logger.Error(directive.Location, $"Directive not recognized: {directiveName}"); + p.IgnoreRestOfLine(tokens); } - - return new Nothing(); } } } diff --git a/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs b/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs new file mode 100644 index 0000000..7afdb96 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs @@ -0,0 +1,55 @@ + +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + public abstract class BaseIncludeDirective : SimpleDirective + { + public override int MinParams => 1; + + public override int? MaxParams => 1; + + public override bool RequireInclusion => true; + + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); + + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + string pathExpression = parameters[0].ToString()!; + + if (EAOptions.TranslateBackslashesInPaths) + { + pathExpression = pathExpression.Replace('\\', '/'); + } + + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); + + if (existantFile != null) + { + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) + { + string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); + + if (pathExpression != portablePathExpression) + { + p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + } + } + + HandleInclude(p, self, existantFile, tokens); + } + else + { + p.Logger.Error(parameters[0].MyLocation, $"Could not find file \"{pathExpression}\"."); + } + } + + public abstract void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens); + } +} diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 83e86f9..815f5ab 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -1,173 +1,125 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Macros; +using ColorzCore.Preprocessor.Macros; namespace ColorzCore.Preprocessor.Directives { class DefineDirective : IDirective { - public int MinParams => 1; + public bool RequireInclusion => true; - public int? MaxParams => 2; + public void Execute(EAParser p, Token self, MergeableGenerator tokens) + { + Token nextToken = tokens.Current; + IList? parameters; - public bool RequireInclusion => true; + switch (nextToken.Type) + { + case TokenType.IDENTIFIER: + tokens.MoveNext(); + parameters = null; + break; + + case TokenType.MAYBE_MACRO: + tokens.MoveNext(); + parameters = FlattenParameters(p, p.ParseMacroParamList(tokens)); + break; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + case TokenType.NEWLINE: + p.Logger.Error(self.Location, "Invalid use of directive '#define': missing macro name."); + return; + + default: + p.Logger.Error(self.Location, $"Invalid use of directive '#define': expected macro name, got {nextToken}"); + p.IgnoreRestOfLine(tokens); + return; + } + + IList? macroBody = ExpandMacroBody(p, p.GetRestOfLine(tokens)); + + if (parameters != null) + { + // function-like macro + DefineFunctionMacro(p, nextToken, parameters, macroBody); + } + else + { + // object-like macro + DefineObjectMacro(p, nextToken, macroBody); + } + } + + private static void DefineObjectMacro(EAParser p, Token nameToken, IList macroBody) { - if (parameters[0].Type == ParamType.MACRO) + string name = nameToken.Content; + + if (p.Definitions.ContainsKey(name) && EAOptions.IsWarningEnabled(EAOptions.Warnings.ReDefine)) { - MacroInvocationNode signature = (MacroInvocationNode)(parameters[0]); - string name = signature.Name; - IList myParams = new List(); - foreach (IList l1 in signature.Parameters) - { - if (l1.Count != 1 || l1[0].Type != TokenType.IDENTIFIER) - { - p.Error(l1[0].Location, "Macro parameters must be identifiers (got " + l1[0].Content + ")."); - } - else - { - myParams.Add(l1[0]); - } - } - /* if (!p.IsValidMacroName(name, myParams.Count)) - { - if (p.IsReservedName(name)) - { - p.Error(signature.MyLocation, "Invalid redefinition: " + name); - } - else - p.Warning(signature.MyLocation, "Redefining " + name + '.'); - }*/ - if(p.Macros.HasMacro(name, myParams.Count)) - p.Warning(signature.MyLocation, "Redefining " + name + '.'); - Maybe> toRepl; - if (parameters.Count != 2) - { - toRepl = new Just>(new List()); - } - else - toRepl = ExpandParam(p, parameters[1], myParams.Select((Token t) => t.Content)); - if (!toRepl.IsNothing) - { - p.Macros.AddMacro(new Macro(myParams, toRepl.FromJust), name, myParams.Count); - } + p.Logger.Warning(nameToken.Location, $"Redefining {name}."); + } + + if (macroBody.Count == 1 && macroBody[0].Type == TokenType.IDENTIFIER && macroBody[0].Content == name) + { + /* an object-like macro whose only inner token is a reference to itself is non-productive + * this is: it doesn't participate in macro expansion. */ + + p.Definitions[name] = new Definition(); } else { - //Note [mutually] recursive definitions are handled by Parser expansion. - Maybe maybeIdentifier; - if (parameters[0].Type == ParamType.ATOM && !(maybeIdentifier = ((IAtomNode)parameters[0]).GetIdentifier()).IsNothing) - { - string name = maybeIdentifier.FromJust; - if(p.Definitions.ContainsKey(name)) - p.Warning(parameters[0].MyLocation, "Redefining " + name + '.'); - if (parameters.Count == 2) - { - Maybe> toRepl = ExpandParam(p, parameters[1], Enumerable.Empty()); - if (!toRepl.IsNothing) - { - p.Definitions[name] = new Definition(toRepl.FromJust); - } - } - else - { - p.Definitions[name] = new Definition(); - } - } - else - { - p.Error(parameters[0].MyLocation, "Definition names must be identifiers (got " + parameters[0].ToString() + ")."); - } + p.Definitions[name] = new Definition(macroBody); } - return new Nothing(); } - delegate Maybe> ExpParamType(EAParser p, IParamNode param, IEnumerable myParams); - private static ExpParamType ExpandParam = (EAParser p, IParamNode param, IEnumerable myParams) => - TokenizeParam(p, param).Fmap>( (IList l) => - ExpandAllIdentifiers(p, new Queue(l), ImmutableStack.FromEnumerable(myParams), ImmutableStack>.Nil)).Fmap( - (IEnumerable x) => (IList)new List(x)); - private static Maybe> TokenizeParam(EAParser p, IParamNode param) + + private static void DefineFunctionMacro(EAParser p, Token nameToken, IList parameters, IList macroBody) { + string name = nameToken.Content; - switch (param.Type) + if (p.Macros.HasMacro(name, parameters.Count) && EAOptions.IsWarningEnabled(EAOptions.Warnings.ReDefine)) { - case ParamType.STRING: - Token input = ((StringNode)param).MyToken; - Tokenizer t = new Tokenizer(); - return new Just>(new List(t.TokenizeLine(input.Content, input.FileName, input.LineNumber, input.ColumnNumber))); - case ParamType.MACRO: - try - { - IList myBody = new List(((MacroInvocationNode)param).ExpandMacro()); - return new Just>(myBody); - } - catch (KeyNotFoundException) - { - MacroInvocationNode asMacro = (MacroInvocationNode)param; - p.Error(asMacro.MyLocation, "Undefined macro: " + asMacro.Name); - } - break; - case ParamType.LIST: - ListNode n = (ListNode)param; - return new Just>(new List(n.ToTokens())); - case ParamType.ATOM: - return new Just>(new List(((IAtomNode)param).ToTokens())); + p.Logger.Warning(nameToken.Location, $"Redefining {name}(...) with {parameters.Count} parameters."); } - return new Nothing>(); + + p.Macros.AddMacro(new UserMacro(parameters, macroBody), name, parameters.Count); } - private static IEnumerable ExpandAllIdentifiers(EAParser p, Queue tokens, ImmutableStack seenDefs, ImmutableStack> seenMacros) + + private static IList FlattenParameters(EAParser p, IList> rawParameters) { - IEnumerable output = new List(); - while(tokens.Count > 0) + IList result = new List(); + + foreach (IList parameter in rawParameters) { - Token current = tokens.Dequeue(); - if(current.Type == TokenType.IDENTIFIER) + if (parameter.Count != 1 || parameter[0].Type != TokenType.IDENTIFIER) { - if(p.Macros.ContainsName(current.Content) && tokens.Count > 0 && tokens.Peek().Type == TokenType.OPEN_PAREN) - { - IList> param = p.ParseMacroParamList(new MergeableGenerator(tokens)); //TODO: I don't like wrapping this in a mergeable generator..... Maybe interface the original better? - if (!seenMacros.Contains(new Tuple(current.Content, param.Count)) && p.Macros.HasMacro(current.Content, param.Count)) - { - foreach(Token t in p.Macros.GetMacro(current.Content, param.Count).ApplyMacro(current, param, p.GlobalScope)) - { - yield return t; - } - } - else if(seenMacros.Contains(new Tuple(current.Content, param.Count))) - { - yield return current; - foreach (IList l in param) - foreach (Token t in l) - yield return t; - } - else - { - yield return current; - } - } - else if(!seenDefs.Contains(current.Content) && p.Definitions.ContainsKey(current.Content)) - { - foreach (Token t in p.Definitions[current.Content].ApplyDefinition(current)) - yield return t; - } - else - { - yield return current; - } - } + p.Logger.Error(parameter[0].Location, $"Macro parameters must be single identifiers (got {parameter[0].Content})."); + result.Add($"${result.Count}"); + } else { - yield return current; + result.Add(parameter[0].Content); } } + + return result; + } + + private static IList ExpandMacroBody(EAParser _, IList body) + { + if (body.Count == 1 && body[0].Type == TokenType.STRING) + { + // FIXME: for some reason, locations of tokens in this are offset by 1 + + Token token = body[0]; + return new List(Tokenizer.TokenizeLine(token.Content, token.Location)); + } + + return body; } } } diff --git a/ColorzCore/Preprocessor/Directives/ElseDirective.cs b/ColorzCore/Preprocessor/Directives/ElseDirective.cs index 7e169d2..1d03012 100644 --- a/ColorzCore/Preprocessor/Directives/ElseDirective.cs +++ b/ColorzCore/Preprocessor/Directives/ElseDirective.cs @@ -1,30 +1,32 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { - class ElseDirective : IDirective + class ElseDirective : SimpleDirective { - public int MinParams => 0; + public override int MinParams => 0; - public int? MaxParams => 0; + public override int? MaxParams => 0; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) - p.Error(self.Location, "No matching if[n]def."); + { + p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); + } else + { p.Inclusion = new ImmutableStack(!p.Inclusion.Head, p.Inclusion.Tail); - return new Nothing(); + } } } } diff --git a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs index 09212a4..84ec8f9 100644 --- a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs @@ -1,30 +1,32 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { - class EndIfDirective : IDirective + class EndIfDirective : SimpleDirective { - public int MinParams => 0; + public override int MinParams => 0; - public int? MaxParams => 0; + public override int? MaxParams => 0; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) - p.Error(self.Location, "No matching if[n]def."); + { + p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); + } else + { p.Inclusion = p.Inclusion.Tail; - return new Nothing(); + } } } } diff --git a/ColorzCore/Preprocessor/Directives/IDirective.cs b/ColorzCore/Preprocessor/Directives/IDirective.cs index 2079dff..730249b 100644 --- a/ColorzCore/Preprocessor/Directives/IDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IDirective.cs @@ -1,16 +1,13 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Preprocessor.Directives { - interface IDirective + public interface IDirective { /*** * Perform the directive's action, be it altering tokens, for just emitting a special ILineNode. @@ -18,15 +15,8 @@ interface IDirective * * Return: If a string is returned, it is interpreted as an error. */ - Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); - /*** - * Minimum number of parameters, inclusive. - */ - int MinParams { get; } - /*** - * Maximum number of parameters, inclusive. Null for no limit. - */ - int? MaxParams { get; } + void Execute(EAParser p, Token self, MergeableGenerator tokens); + /*** * Whether requires the parser to be taking in tokens. * This may not hold when the parser is skipping, e.g. from an #ifdef. diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index b4e9144..cd871be 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -1,40 +1,51 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { class IfDefinedDirective : IDirective { - public int MinParams => 1; - - public int? MaxParams => 1; + // This directive does not inherit SimpleDirective so as to avoid having its parameter expanded public bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public bool Inverted { get; } + + public IfDefinedDirective(bool invert) { - bool flag = true; - Maybe identifier; - foreach (IParamNode parameter in parameters) + Inverted = invert; + } + + public void Execute(EAParser p, Token self, MergeableGenerator tokens) + { + if (tokens.Current.Type != TokenType.IDENTIFIER) { - if(parameter.Type==ParamType.ATOM && !(identifier = ((IAtomNode)parameter).GetIdentifier()).IsNothing) - { - flag &= p.Macros.ContainsName(identifier.FromJust) || p.Definitions.ContainsKey(identifier.FromJust); //TODO: Built in definitions? - } - else + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected macro name, got {tokens.Current}."); + p.IgnoreRestOfLine(tokens); + } + else + { + string identifier = tokens.Current.Content; + tokens.MoveNext(); + + // here we could parse a potential parameter list/specifier + + bool isDefined = p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); + bool flag = Inverted ? !isDefined : isDefined; + + p.Inclusion = new ImmutableStack(flag, p.Inclusion); + + if (tokens.Current.Type != TokenType.NEWLINE) { - p.Error(parameter.MyLocation, "Definition name must be an identifier."); + p.Logger.Error(self.Location, $"Garbage at the end of directive '{self.Content}' (got {tokens.Current})."); + p.IgnoreRestOfLine(tokens); } } - p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return new Nothing(); } } } diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs new file mode 100644 index 0000000..459b24c --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + class IfDirective : SimpleDirective + { + public override int MinParams => 1; + + public override int? MaxParams => 1; + + public override bool RequireInclusion => false; + + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + bool flag = true; + + foreach (IParamNode parameter in parameters) + { + if (parameter is IAtomNode atomNode) + { + if (atomNode.TryEvaluate(e => p.Logger.Error(self.Location, $"Error while evaluating expression: {e.Message}"), EvaluationPhase.Immediate) is int value) + { + flag = value != 0; + } + } + else + { + p.Logger.Error(self.Location, "Expected an expression."); + } + } + + p.Inclusion = new ImmutableStack(flag, p.Inclusion); + } + } +} diff --git a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs deleted file mode 100644 index 02476b1..0000000 --- a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.DataTypes; -using ColorzCore.Lexer; -using ColorzCore.Parser; -using ColorzCore.Parser.AST; - -namespace ColorzCore.Preprocessor.Directives -{ - class IfNotDefinedDirective : IDirective - { - public int MinParams => 1; - - public int? MaxParams => 1; - - public bool RequireInclusion => false; - - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) - { - bool flag = true; - Maybe identifier; - foreach (IParamNode parameter in parameters) - { - if(parameter.Type==ParamType.ATOM && !(identifier = ((IAtomNode)parameter).GetIdentifier()).IsNothing) - { - flag &= !p.Macros.ContainsName(identifier.FromJust) && !p.Definitions.ContainsKey(identifier.FromJust); //TODO: Built in definitions? - } - else - { - p.Error(parameter.MyLocation, "Definition name must be an identifier."); - } - } - p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return new Nothing(); - } - } -} diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index c590ed3..2c5b5f1 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -1,48 +1,28 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser.AST; using System.IO; using ColorzCore.Parser; -using ColorzCore.IO; - +using ColorzCore.IO; + namespace ColorzCore.Preprocessor.Directives { - class IncludeBinaryDirective : IDirective + public class IncludeBinaryDirective : BaseIncludeDirective { - public int MinParams { get { return 1; } } - - public int? MaxParams { get { return 1; } } - - public bool RequireInclusion { get { return true; } } - - public IncludeFileSearcher FileSearcher { get; set; } - - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator _) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()); - - if (!existantFile.IsNothing) + try { - try - { - string pathname = existantFile.FromJust; - return new Just(new DataNode(p.CurrentOffset, File.ReadAllBytes(pathname))); - } - catch (Exception) - { - p.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); - } + byte[] data = File.ReadAllBytes(path); + p.ParseConsumer.OnData(self.Location, data); } - else + catch (IOException e) { - p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); + p.Logger.Error(self.Location, $"Error reading file \"{path}\": {e.Message}."); } - return new Nothing(); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index e70f6c3..3b4f653 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -1,51 +1,28 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser.AST; using System.IO; using ColorzCore.Parser; -using ColorzCore.IO; - +using ColorzCore.IO; + namespace ColorzCore.Preprocessor.Directives { - class IncludeDirective : IDirective + class IncludeDirective : BaseIncludeDirective { - public int MinParams { get { return 1; } } - - public int? MaxParams { get { return 1; } } - - public bool RequireInclusion { get { return true; } } - - public IncludeFileSearcher FileSearcher { get; set; } - - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()); - - if (!existantFile.IsNothing) + try { - try - { - string pathname = existantFile.FromJust; - - FileStream inputFile = new FileStream(pathname, FileMode.Open); - Tokenizer newFileTokenizer = new Tokenizer(); - tokens.PrependEnumerator(newFileTokenizer.Tokenize(inputFile).GetEnumerator()); - } - catch(Exception) - { - p.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); - } + FileStream inputFile = new FileStream(path, FileMode.Open); + tokens.PrependEnumerator(new Tokenizer().TokenizeFile(inputFile, path.Replace('\\', '/')).GetEnumerator()); } - else + catch (IOException e) { - p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); + p.Logger.Error(self.Location, $"Error reading file \"{path}\": {e.Message}."); } - return new Nothing(); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs new file mode 100644 index 0000000..e9263e0 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + public class IncludeEncodingTableDirective : BaseIncludeDirective + { + private readonly StringProcessor stringProcessor; + + public IncludeEncodingTableDirective(StringProcessor stringProcessor) + { + this.stringProcessor = stringProcessor; + } + + public override int MinParams => 2; + + public override int? MaxParams => 2; + + /* HACK: this is set by HandleInclude and read by Execute + * because we depend BaseIncludeDirective Execute, there's no elegant way of passing this around. */ + private TblEncoding? lastTblEncoding = null; + + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + string? encodingName = parameters[0].ToString(); + + if (encodingName != null) + { + if (stringProcessor.TableEncodings.ContainsKey(encodingName)) + { + p.Logger.Error(self.Location, $"String encoding '{encodingName}' already exists."); + } + else + { + base.Execute(p, self, new List() { parameters[1] }, tokens); + + if (lastTblEncoding != null) + { + stringProcessor.TableEncodings[encodingName] = lastTblEncoding; + lastTblEncoding = null; + } + else + { + p.Logger.Error(self.Location, $"Could not load encoding from table file '{parameters[1]}'."); + } + } + } + else + { + p.Logger.Error(self.Location, $"{self.Content} expected encoding name as first parameter."); + } + } + + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens) + { + using TextReader textReader = new StreamReader(path); + lastTblEncoding = TblEncoding.FromTextReader(textReader); + } + } +} diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index cb82495..522bdcc 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -2,34 +2,34 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.IO; using System.IO; +using ColorzCore.Interpreter; namespace ColorzCore.Preprocessor.Directives { - class IncludeExternalDirective : IDirective + class IncludeExternalDirective : SimpleDirective { - public int MinParams { get { return 1; } } - public int? MaxParams { get { return null; } } - public bool RequireInclusion { get { return true; } } + public override int MinParams => 1; + public override int? MaxParams => null; + public override bool RequireInclusion => true; - public IncludeFileSearcher FileSearcher { get; set; } + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString())); + string? validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); - if (validFile.IsNothing) + if (validFile == null) { - parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return new Nothing(); + parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); + return; } //TODO: abstract out all this running stuff into a method so I don't have code duplication with inctext @@ -38,17 +38,17 @@ public Maybe Execute(EAParser parse, Token self, IList pa System.Diagnostics.Process p = new System.Diagnostics.Process(); p.StartInfo.RedirectStandardError = true; // Redirect the output stream of the child process. - p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName); + p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName)!; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; - p.StartInfo.FileName = validFile.FromJust; + p.StartInfo.FileName = validFile; StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { - if(parameters[i].Type == ParamType.ATOM) + if (parameters[i].Type == ParamType.ATOM) { - parameters[i] = ((IAtomNode)parameters[i]).Simplify(); + parameters[i] = ((IAtomNode)parameters[i]).Simplify(EvaluationPhase.Early); } argumentBuilder.Append(parameters[i].PrettyPrint()); argumentBuilder.Append(' '); @@ -69,16 +69,16 @@ public Maybe Execute(EAParser parse, Token self, IList pa byte[] output = outputBytes.GetBuffer().Take((int)outputBytes.Length).ToArray(); if (errorStream.Length > 0) { - parse.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); } else if (output.Length >= 7 && Encoding.ASCII.GetString(output.Take(7).ToArray()) == "ERROR: ") { - parse.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); } - ExecTimer.Timer.AddTimingPoint(parameters[0].ToString().ToLower()); + ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - return new Just(new DataNode(parse.CurrentOffset, output)); + parse.ParseConsumer.OnData(self.Location, output); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 14111a4..531ffcb 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -2,34 +2,34 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.IO; using System.IO; +using ColorzCore.Interpreter; namespace ColorzCore.Preprocessor.Directives { - class IncludeToolEventDirective : IDirective + class IncludeToolEventDirective : SimpleDirective { - public int MinParams { get { return 1; } } - public int? MaxParams { get { return null; } } - public bool RequireInclusion { get { return true; } } + public override int MinParams => 1; + public override int? MaxParams => null; + public override bool RequireInclusion => true; - public IncludeFileSearcher FileSearcher { get; set; } + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString())); + string? validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); - if (validFile.IsNothing) + if (validFile == null) { - parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return new Nothing(); + parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); + return; } //from http://stackoverflow.com/a/206347/1644720 @@ -37,15 +37,19 @@ public Maybe Execute(EAParser parse, Token self, IList pa System.Diagnostics.Process p = new System.Diagnostics.Process(); p.StartInfo.RedirectStandardError = true; // Redirect the output stream of the child process. - p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName); + p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName)!; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; - p.StartInfo.FileName = validFile.FromJust; + p.StartInfo.FileName = validFile; StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { - parameters[i].AsAtom().IfJust((IAtomNode n) => { parameters[i] = n.Simplify(); }); + if (parameters[i] is IAtomNode atom) + { + parameters[i] = atom.Simplify(EvaluationPhase.Early); + } + argumentBuilder.Append(parameters[i].PrettyPrint()); argumentBuilder.Append(' '); } @@ -63,13 +67,13 @@ public Maybe Execute(EAParser parse, Token self, IList pa p.WaitForExit(); byte[] output = outputBytes.GetBuffer().Take((int)outputBytes.Length).ToArray(); - if(errorStream.Length > 0) + if (errorStream.Length > 0) { - parse.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); } else if (output.Length >= 7 && Encoding.ASCII.GetString(output.Take(7).ToArray()) == "ERROR: ") { - parse.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); } else { @@ -78,9 +82,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa tokens.PrependEnumerator(t.Tokenize(outputBytes, Path.GetFileName(self.FileName) + ", Line " + self.LineNumber + "; " + parameters[0].ToString()).GetEnumerator()); } - ExecTimer.Timer.AddTimingPoint(parameters[0].ToString().ToLower()); - - return new Nothing(); + ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); } } } diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index fbd057c..4e98ecb 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -7,36 +7,38 @@ namespace ColorzCore.Preprocessor.Directives { - class PoolDirective : IDirective + class PoolDirective : SimpleDirective { - public int MinParams => 0; - public int? MaxParams => 0; - public bool RequireInclusion => true; + public override int MinParams => 0; + public override int? MaxParams => 0; + public override bool RequireInclusion => true; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + private readonly Pool pool; + + public PoolDirective(Pool pool) { - BlockNode result = new BlockNode(); + this.pool = pool; + } + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { // Iterating indices (and not values via foreach) // to avoid crashes occuring with AddToPool within AddToPool - for (int i = 0; i < p.Pool.Lines.Count; ++i) + for (int i = 0; i < pool.Lines.Count; ++i) { - Pool.PooledLine line = p.Pool.Lines[i]; + Pool.PooledLine line = pool.Lines[i]; MergeableGenerator tempGenerator = new MergeableGenerator(line.Tokens); tempGenerator.MoveNext(); while (!tempGenerator.EOS) { - p.ParseLine(tempGenerator, line.Scope).IfJust( - (lineNode) => result.Children.Add(lineNode)); + p.ParseLine(tempGenerator); } } - p.Pool.Lines.Clear(); - - return new Just(result); + pool.Lines.Clear(); } } } diff --git a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs new file mode 100644 index 0000000..b895a7c --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + /*** + * Simple abstract base class for directives that don't care about the details of their parameters + */ + public abstract class SimpleDirective : IDirective + { + public abstract bool RequireInclusion { get; } + + /*** + * Minimum number of parameters, inclusive. + */ + public abstract int MinParams { get; } + + /*** + * Maximum number of parameters, inclusive. Null for no limit. + */ + public abstract int? MaxParams { get; } + + public void Execute(EAParser p, Token self, MergeableGenerator tokens) + { + // Note: Not a ParseParamList because no commas. + // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not + IList parameters = p.ParsePreprocParamList(tokens); + + if (MinParams <= parameters.Count && (!MaxParams.HasValue || parameters.Count <= MaxParams)) + { + Execute(p, self, parameters, tokens); + } + else + { + p.Logger.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self.Content}."); + } + } + + public abstract void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); + } +} diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 287fa26..21a1c4e 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -1,34 +1,51 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { class UndefineDirective : IDirective { - public int MinParams => 1; + public bool RequireInclusion => true; - public int? MaxParams => null; + public void Execute(EAParser p, Token self, MergeableGenerator tokens) + { + if (tokens.Current.Type == TokenType.NEWLINE) + { + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected at least one macro name."); + } - public bool RequireInclusion => true; + while (tokens.Current.Type != TokenType.NEWLINE) + { + Token current = tokens.Current; + tokens.MoveNext(); - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + switch (current.Type) + { + case TokenType.IDENTIFIER: + ApplyUndefine(p, current); + break; + + default: + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected macro name, got {current}."); + p.IgnoreRestOfLine(tokens); + return; + } + } + } + + private static void ApplyUndefine(EAParser parser, Token token) { - foreach (IParamNode parm in parameters) + string name = token.Content; + + if (!parser.Definitions.Remove(name)) { - string s = parm.ToString(); - if (p.Definitions.ContainsKey(s)) - p.Definitions.Remove(parm.ToString()); - else - p.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); + parser.Logger.Warning(token.Location, $"Attempted to purge non existant definition '{name}'"); } - return new Nothing(); } } } diff --git a/ColorzCore/Parser/Macros/MacroCollection.cs b/ColorzCore/Preprocessor/MacroCollection.cs similarity index 64% rename from ColorzCore/Parser/Macros/MacroCollection.cs rename to ColorzCore/Preprocessor/MacroCollection.cs index 40e78d0..6d0c58b 100644 --- a/ColorzCore/Parser/Macros/MacroCollection.cs +++ b/ColorzCore/Preprocessor/MacroCollection.cs @@ -1,12 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; +using ColorzCore.Parser; +using ColorzCore.Preprocessor.Macros; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor { - class MacroCollection + public class MacroCollection { public Dictionary BuiltInMacros { get; } public EAParser Parent { get; } @@ -18,10 +18,10 @@ public MacroCollection(EAParser parent) Macros = new Dictionary>(); Parent = parent; - BuiltInMacros = new Dictionary { - { "String", new String() }, + BuiltInMacros = new Dictionary + { + { "String", new StringMacro() }, { "IsDefined", new IsDefined(parent) }, - { "AddToPool", new AddToPool(parent) }, }; } @@ -29,16 +29,37 @@ public bool HasMacro(string name, int paramNum) { return BuiltInMacros.ContainsKey(name) && BuiltInMacros[name].ValidNumParams(paramNum) || Macros.ContainsKey(name) && Macros[name].ContainsKey(paramNum); } + + // NOTE: NotNullWhen(true) is not available on .NET Framework 4.x, one of our targets + public bool TryGetMacro(string name, int paramNum, out IMacro? macro) + { + if (BuiltInMacros.TryGetValue(name, out BuiltInMacro? builtinMacro) && builtinMacro.ValidNumParams(paramNum)) + { + macro = builtinMacro; + return true; + } + + if (Macros.TryGetValue(name, out Dictionary? macros)) + { + return macros.TryGetValue(paramNum, out macro); + } + + macro = null; + return false; + } + public IMacro GetMacro(string name, int paramNum) { return BuiltInMacros.ContainsKey(name) && BuiltInMacros[name].ValidNumParams(paramNum) ? BuiltInMacros[name] : Macros[name][paramNum]; } + public void AddMacro(IMacro macro, string name, int paramNum) { if (!Macros.ContainsKey(name)) Macros[name] = new Dictionary(); Macros[name][paramNum] = macro; } + public void Clear() { Macros.Clear(); diff --git a/ColorzCore/Parser/Macros/AddToPool.cs b/ColorzCore/Preprocessor/Macros/AddToPool.cs similarity index 80% rename from ColorzCore/Parser/Macros/AddToPool.cs rename to ColorzCore/Preprocessor/Macros/AddToPool.cs index 565b265..5b98b80 100644 --- a/ColorzCore/Parser/Macros/AddToPool.cs +++ b/ColorzCore/Preprocessor/Macros/AddToPool.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { class AddToPool : BuiltInMacro { @@ -13,18 +14,18 @@ class AddToPool : BuiltInMacro * AddToPool(tokens..., alignment): adds token to pool and make sure pooled tokens are aligned given alignment */ - public EAParser ParentParser { get; private set; } + public Pool Pool { get; } - public AddToPool(EAParser parent) + public AddToPool(Pool pool) { - ParentParser = parent; + Pool = pool; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { List line = new List(6 + parameters[0].Count); - string labelName = ParentParser.Pool.MakePoolLabelName(); + string labelName = Pool.MakePoolLabelName(); if (parameters.Count == 2) { @@ -44,7 +45,7 @@ public override IEnumerable ApplyMacro(Token head, IList> pa line.AddRange(parameters[0]); line.Add(new Token(TokenType.NEWLINE, head.Location, "\n")); - ParentParser.Pool.Lines.Add(new Pool.PooledLine(scopes, line)); + Pool.Lines.Add(new Pool.PooledLine(line)); yield return new Token(TokenType.IDENTIFIER, head.Location, labelName); } diff --git a/ColorzCore/Parser/Macros/BuiltInMacro.cs b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs similarity index 57% rename from ColorzCore/Parser/Macros/BuiltInMacro.cs rename to ColorzCore/Preprocessor/Macros/BuiltInMacro.cs index a38209d..31b97b3 100644 --- a/ColorzCore/Parser/Macros/BuiltInMacro.cs +++ b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs @@ -1,16 +1,15 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { - abstract class BuiltInMacro : IMacro + public abstract class BuiltInMacro : IMacro { public abstract bool ValidNumParams(int num); - public abstract IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes); + public abstract IEnumerable ApplyMacro(Token head, IList> parameters); } } diff --git a/ColorzCore/Preprocessor/Macros/ErrorMacro.cs b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs new file mode 100644 index 0000000..460f710 --- /dev/null +++ b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs @@ -0,0 +1,34 @@ +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ColorzCore.Preprocessor.Macros +{ + class ErrorMacro : BuiltInMacro + { + public delegate bool ValidateNumParamsIndirect(int num); + + private readonly EAParser parser; + private readonly string message; + private readonly ValidateNumParamsIndirect validateNumParamsIndirect; + + public ErrorMacro(EAParser parser, string message, ValidateNumParamsIndirect validateNumParamsIndirect) + { + this.parser = parser; + this.message = message; + this.validateNumParamsIndirect = validateNumParamsIndirect; + } + + public override IEnumerable ApplyMacro(Token head, IList> parameters) + { + parser.Logger.Error(head.Location, message); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + + public override bool ValidNumParams(int num) => validateNumParamsIndirect(num); + } +} diff --git a/ColorzCore/Parser/Macros/IMacro.cs b/ColorzCore/Preprocessor/Macros/IMacro.cs similarity index 76% rename from ColorzCore/Parser/Macros/IMacro.cs rename to ColorzCore/Preprocessor/Macros/IMacro.cs index 0cbd03f..1aa11cf 100644 --- a/ColorzCore/Parser/Macros/IMacro.cs +++ b/ColorzCore/Preprocessor/Macros/IMacro.cs @@ -1,15 +1,16 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { public interface IMacro { - IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes); + IEnumerable ApplyMacro(Token head, IList> parameters); } } diff --git a/ColorzCore/Parser/Macros/IsDefined.cs b/ColorzCore/Preprocessor/Macros/IsDefined.cs similarity index 85% rename from ColorzCore/Parser/Macros/IsDefined.cs rename to ColorzCore/Preprocessor/Macros/IsDefined.cs index 135c840..2cd6218 100644 --- a/ColorzCore/Parser/Macros/IsDefined.cs +++ b/ColorzCore/Preprocessor/Macros/IsDefined.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { class IsDefined : BuiltInMacro { @@ -14,7 +15,7 @@ public IsDefined(EAParser parent) ParentParser = parent; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { if (parameters[0].Count != 1) { @@ -46,12 +47,12 @@ protected bool IsReallyDefined(string name) return ParentParser.Definitions.ContainsKey(name) || ParentParser.Macros.ContainsName(name); } - protected static Token MakeTrueToken(DataTypes.Location location) + protected static Token MakeTrueToken(Location location) { return new Token(TokenType.NUMBER, location, "1"); } - protected static Token MakeFalseToken(DataTypes.Location location) + protected static Token MakeFalseToken(Location location) { return new Token(TokenType.NUMBER, location, "0"); } diff --git a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs new file mode 100644 index 0000000..5cbeaa7 --- /dev/null +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -0,0 +1,74 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Macros +{ + public class ReadDataAt : BuiltInMacro + { + private readonly EAParser parser; + private readonly ROM rom; + private readonly int readLength; + + public ReadDataAt(EAParser parser, ROM rom, int readLength) + { + this.parser = parser; + this.rom = rom; + this.readLength = readLength; + } + + public override IEnumerable ApplyMacro(Token head, IList> parameters) + { + // HACK: hack + MergeableGenerator tokens = new MergeableGenerator( + Enumerable.Repeat(new Token(TokenType.NEWLINE, head.Location, "\n"), 1)); + tokens.PrependEnumerator(parameters[0].GetEnumerator()); + + IAtomNode? atom = parser.ParseAtom(tokens); + + if (tokens.Current.Type != TokenType.NEWLINE) + { + parser.Logger.Error(head.Location, "Garbage at the end of macro parameter."); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + else if (atom?.TryEvaluate(e => parser.Logger.Error(atom.MyLocation, e.Message), EvaluationPhase.Immediate) is int offset) + { + offset = EAInterpreter.ConvertToOffset(offset); + + if (offset >= 0 && offset <= EAOptions.MaximumBinarySize - readLength) + { + int data = 0; + + // little endian!!! + for (int i = 0; i < readLength; i++) + { + data |= rom[offset + i] << (i * 8); + } + + yield return new Token(TokenType.NUMBER, head.Location, $"0x{data:X}"); + } + else + { + parser.Logger.Error(head.Location, $"Read offset out of bounds: {offset:08X}"); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + } + else + { + parser.Logger.Error(head.Location, "Could not read data from base binary."); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + } + + public override bool ValidNumParams(int num) + { + return num == 1; + } + } +} diff --git a/ColorzCore/Preprocessor/Macros/StringMacro.cs b/ColorzCore/Preprocessor/Macros/StringMacro.cs new file mode 100644 index 0000000..88b0cd4 --- /dev/null +++ b/ColorzCore/Preprocessor/Macros/StringMacro.cs @@ -0,0 +1,38 @@ +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace ColorzCore.Preprocessor.Macros +{ + class StringMacro : BuiltInMacro + { + public override IEnumerable ApplyMacro(Token head, IList> parameters) + { + MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); + + Token token = parameters[0][0]; + Location location = token.Location.MacroClone(macroLocation); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.LegacyFeatures)) + { + yield return new Token(TokenType.IDENTIFIER, location, "WARNING"); + yield return new Token(TokenType.STRING, location, "Consider using the STRING statement rather than the legacy String macro."); + yield return new Token(TokenType.SEMICOLON, location, ";"); + } + + yield return new Token(TokenType.IDENTIFIER, location, "STRING"); + yield return new Token(TokenType.STRING, location, token.Content); + yield return new Token(TokenType.STRING, location, "UTF-8"); + yield return new Token(TokenType.SEMICOLON, location, ";"); + } + + public override bool ValidNumParams(int num) + { + return num == 1; + } + } +} diff --git a/ColorzCore/Preprocessor/Macros/UserMacro.cs b/ColorzCore/Preprocessor/Macros/UserMacro.cs new file mode 100644 index 0000000..e5e8d74 --- /dev/null +++ b/ColorzCore/Preprocessor/Macros/UserMacro.cs @@ -0,0 +1,50 @@ +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ColorzCore.Preprocessor.Macros +{ + class UserMacro : IMacro + { + readonly Dictionary idToParamNum; + readonly IList body; + + public UserMacro(IList parameters, IList macroBody) + { + idToParamNum = new Dictionary(); + + for (int i = 0; i < parameters.Count; i++) + { + idToParamNum[parameters[i]] = i; + } + + body = macroBody; + } + + /*** + * Precondition: parameters.Count = max(keys(idToParamNum)) + */ + public IEnumerable ApplyMacro(Token head, IList> parameters) + { + MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); + + foreach (Token bodyToken in body) + { + if (bodyToken.Type == TokenType.IDENTIFIER && idToParamNum.TryGetValue(bodyToken.Content, out int paramNum)) + { + foreach (Token paramToken in parameters[paramNum]) + { + yield return paramToken.MacroClone(macroLocation); + } + } + else + { + yield return bodyToken.MacroClone(macroLocation); + } + } + } + } +} diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 99add64..6e7ffbc 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -10,7 +10,18 @@ class Program { public static bool Debug = false; - private static string[] helpstringarr = { + private static readonly IDictionary warningNames = new Dictionary() + { + { "nonportable-pathnames", EAOptions.Warnings.NonPortablePath }, + { "unintuitive-expression-macros" , EAOptions.Warnings.UnintuitiveExpressionMacros }, + { "unguarded-expression-macros", EAOptions.Warnings.UnguardedExpressionMacros }, + { "redefine", EAOptions.Warnings.ReDefine }, + { "legacy", EAOptions.Warnings.LegacyFeatures }, + { "all", EAOptions.Warnings.All }, + { "extra", EAOptions.Warnings.Extra }, + }; + + private static readonly string[] helpstringarr = { "EA Colorz Core. Usage:", "./ColorzCore [-opts]", "", @@ -38,6 +49,12 @@ class Program " Add given path to list of paths to search for tools in.", "-IT:|-TI:", " Combines --include: and --tools:.", + "-W:[no-]:...|--warnings:[no-]:...", + " Enable or disable warnings.", + " By default, all warnings but 'unguarded-expression-macros' are enabled.", + " Multiple warnings can be enabled/disabled at once.", + " Example: '--warnings:no-nonportable-pathnames:no-redefine'.", + " Possible values: " + string.Join(", ", warningNames.Keys), "-werr", " Treat all warnings as errors and prevent assembly.", "--no-mess", @@ -54,10 +71,14 @@ class Program " Enable debug mode. Not recommended for end users.", "--build-times", " Print build times at the end of build.", + "--base-address:", + " Treats the base load address of the binary as the given (hexadecimal) number,", + " for the purposes of POIN, ORG and CURRENTOFFSET. Defaults to 0x08000000.", + " Addresses are added to offsets from 0 to the maximum binary size.", + "--maximum-size:", + " Sets the maximum size of the binary. Defaults to 0x02000000.", "-romoffset:", - " Treats the offset of the ROM as the given number,", - " for the purposes of POIN. Addresses are or'd.", - " Hex literals only. Defaults to 0x08000000.", + " Compatibility alias for --base-address:", "-h|--help", " Display this message and exit.", "" @@ -77,18 +98,17 @@ static int Main(string[] args) Stream inStream = Console.OpenStandardInput(); string inFileName = "stdin"; - IOutput output = null; string outFileName = "none"; string ldsFileName = "none"; TextWriter errorStream = Console.Error; - Maybe rawsFolder = rawSearcher.FindDirectory("Language Raws"); + string? rawsFolder = rawSearcher.FindDirectory("Language Raws"); string rawsExtension = ".txt"; if (args.Length < 2) { - Console.WriteLine("Required parameters missing."); + Console.WriteLine(helpstring); return EXIT_FAILURE; } @@ -107,210 +127,274 @@ static int Main(string[] args) if (args[i][0] != '-') { Console.Error.WriteLine("Unrecognized paramter: " + args[i]); + continue; } - else - { - string[] flag = args[i].Substring(1).Split(new char[] { ':' }, 2); - try + string[] flag = args[i].Split(new char[] { ':' }, 2); + + try + { + switch (flag[0]) { - switch (flag[0]) - { - case "raws": - rawsFolder = rawSearcher.FindDirectory(flag[1]); - break; - - case "rawsExt": - rawsExtension = flag[1]; - break; - - case "output": - outFileName = flag[1]; - if(outputASM) - { - ldsFileName = Path.ChangeExtension(outFileName, "lds"); - output = new ASM(new StreamWriter(outFileName, false), - new StreamWriter(ldsFileName, false)); - } else + case "-raws": + rawsFolder = rawSearcher.FindDirectory(flag[1]); + + if (rawsFolder == null) + { + Console.Error.WriteLine($"No such folder: {flag[1]}"); + return EXIT_FAILURE; + } + + break; + + case "-rawsExt": + rawsExtension = flag[1]; + break; + + case "-output": + outFileName = flag[1]; + break; + + case "-input": + inFileName = flag[1].Replace('\\', '/'); + inStream = File.OpenRead(flag[1]); + break; + + case "-error": + errorStream = new StreamWriter(File.OpenWrite(flag[1])); + EAOptions.MonochromeLog = true; + break; + + case "-debug": + Debug = true; + break; + + case "-werr": + EAOptions.WarningsAreErrors = true; + break; + + case "--no-mess": + EAOptions.QuietMessages = true; + break; + + case "--no-warn": + EAOptions.QuietWarnings = true; + break; + + case "--no-colored-log": + EAOptions.MonochromeLog = true; + break; + + case "-quiet": + case "--quiet": + EAOptions.QuietMessages = true; + EAOptions.QuietWarnings = true; + break; + + case "--nocash-sym": + EAOptions.ProduceNocashSym = true; + break; + + case "--build-times": + EAOptions.BenchmarkBuildTimes = true; + break; + + case "-I": + case "--include": + EAOptions.IncludePaths.Add(flag[1]); + break; + + case "-T": + case "--tools": + EAOptions.ToolsPaths.Add(flag[1]); + break; + + case "-IT": + case "-TI": + EAOptions.IncludePaths.Add(flag[1]); + EAOptions.ToolsPaths.Add(flag[1]); + break; + + case "-h": + case "--help": + Console.Out.WriteLine(helpstring); + return EXIT_SUCCESS; + + case "-D": + case "-def": + case "-define": + try + { + string[] def_args = flag[1].Split(new char[] { '=' }, 2); + EAOptions.PreDefintions.Add((def_args[0], def_args[1])); + } + catch (IndexOutOfRangeException) + { + Console.Error.WriteLine("Improperly formed -define directive."); + } + break; + + case "-romoffset": + case "--base-address": + try + { + EAOptions.BaseAddress = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex base address given for binary."); + } + break; + + case "--maximum-size": + try + { + EAOptions.MaximumBinarySize = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex size given for binary."); + } + break; + + case "-W": + case "--warnings": + if (flag.Length == 1) + { + EAOptions.EnabledWarnings |= EAOptions.Warnings.All; + } + else + { + foreach (string warning in flag[1].Split(':')) { - FileStream outStream; - if(File.Exists(outFileName) && !File.GetAttributes(outFileName).HasFlag(FileAttributes.ReadOnly)) + string name = warning; + bool invert = false; + + if (name.StartsWith("no-")) { - outStream = File.Open(outFileName, FileMode.Open, FileAccess.ReadWrite); - } else if(!File.Exists(outFileName)) + name = name.Substring(3); + invert = true; + } + + if (warningNames.TryGetValue(name, out EAOptions.Warnings warnFlag)) { - outStream = File.Create(outFileName); - } else + if (invert) + { + EAOptions.EnabledWarnings &= ~warnFlag; + } + else + { + EAOptions.EnabledWarnings |= warnFlag; + } + } + else { - Console.Error.WriteLine("Output file is read-only."); - return EXIT_FAILURE; + Console.Error.WriteLine($"Unrecognized warning: {name}"); } - output = new ROM(outStream); - } - break; - - case "input": - inFileName = flag[1]; - inStream = File.OpenRead(flag[1]); - break; - - case "error": - errorStream = new StreamWriter(File.OpenWrite(flag[1])); - EAOptions.Instance.noColoredLog = true; - break; - - case "debug": - Debug = true; - break; - - case "werr": - EAOptions.Instance.werr = true; - break; - - case "-no-mess": - EAOptions.Instance.nomess = true; - break; - - case "-no-warn": - EAOptions.Instance.nowarn = true; - break; - - case "-no-colored-log": - EAOptions.Instance.noColoredLog = true; - break; - - case "quiet": - EAOptions.Instance.nomess = true; - EAOptions.Instance.nowarn = true; - break; - - case "-nocash-sym": - EAOptions.Instance.nocashSym = true; - break; - - case "-build-times": - EAOptions.Instance.buildTimes = true; - break; - - case "I": - case "-include": - EAOptions.Instance.includePaths.Add(flag[1]); - break; - - case "T": - case "-tools": - EAOptions.Instance.toolsPaths.Add(flag[1]); - break; - - case "IT": - case "TI": - EAOptions.Instance.includePaths.Add(flag[1]); - EAOptions.Instance.toolsPaths.Add(flag[1]); - break; - - case "h": - case "-help": - Console.Out.WriteLine(helpstring); - return EXIT_SUCCESS; - - case "D": - case "def": - case "define": - try { - string[] def_args = flag[1].Split(new char[] { '=' }, 2); - EAOptions.Instance.defs.Add(Tuple.Create(def_args[0], def_args[1])); - } catch (IndexOutOfRangeException) - { - Console.Error.WriteLine("Improperly formed -define directive."); } - break; + } - case "romoffset": - try - { - EAOptions.Instance.romOffset = Convert.ToInt32(flag[1], 16); - } catch { - Console.Error.WriteLine("Invalid hex offset given for ROM."); - } - break; + break; - default: - Console.Error.WriteLine("Unrecognized flag: " + flag[0]); - return EXIT_FAILURE; - } - } - catch (IOException e) - { - Console.Error.WriteLine("Exception: " + e.Message); - return EXIT_FAILURE; + default: + Console.Error.WriteLine($"Unrecognized flag: {flag[0]}"); + return EXIT_FAILURE; } } + catch (IOException e) + { + Console.Error.WriteLine("Exception: " + e.Message); + return EXIT_FAILURE; + } } - - if (output == null) + + if (outFileName == null) { Console.Error.WriteLine("No output specified for assembly."); return EXIT_FAILURE; } - if (rawsFolder.IsNothing) + IOutput output; + + if (outputASM) { - Console.Error.WriteLine("Couldn't find raws folder"); - return EXIT_FAILURE; + ldsFileName = Path.ChangeExtension(outFileName, "lds"); + output = new ASM(new StreamWriter(outFileName, false), + new StreamWriter(ldsFileName, false)); + } + else + { + FileStream outStream; + + if (File.Exists(outFileName) && !File.GetAttributes(outFileName).HasFlag(FileAttributes.ReadOnly)) + { + outStream = File.Open(outFileName, FileMode.Open, FileAccess.ReadWrite); + } + else if (!File.Exists(outFileName)) + { + outStream = File.Create(outFileName); + } + else + { + Console.Error.WriteLine("Output file is read-only."); + return EXIT_FAILURE; + } + + output = new ROM(outStream, EAOptions.MaximumBinarySize); } string game = args[1]; //FirstPass(Tokenizer.Tokenize(inputStream)); - Log log = new Log { + Logger log = new Logger + { Output = errorStream, - WarningsAreErrors = EAOptions.Instance.werr, - NoColoredTags = EAOptions.Instance.noColoredLog + WarningsAreErrors = EAOptions.WarningsAreErrors, + NoColoredTags = EAOptions.MonochromeLog, + LocationBasePath = IOUtility.GetPortableBasePathForPrefix(inFileName), }; - if (EAOptions.Instance.nowarn) - log.IgnoredKinds.Add(Log.MsgKind.WARNING); + if (EAOptions.QuietWarnings) + log.IgnoredKinds.Add(Logger.MessageKind.WARNING); - if (EAOptions.Instance.nomess) - log.IgnoredKinds.Add(Log.MsgKind.MESSAGE); + if (EAOptions.QuietMessages) + log.IgnoredKinds.Add(Logger.MessageKind.MESSAGE); - EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder.FromJust, rawsExtension, inStream, inFileName, log); + EADriver myDriver = new EADriver(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_RAWPROC); - bool success = myInterpreter.Interpret(); + bool success = myDriver.Interpret(); - if (success && EAOptions.Instance.nocashSym) + if (success && EAOptions.ProduceNocashSym) { - using (var symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym"))) + using StreamWriter symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym")); + + if (!(success = myDriver.WriteNocashSymbols(symOut))) { - if (!(success = myInterpreter.WriteNocashSymbols(symOut))) - { - log.Message(Log.MsgKind.ERROR, "Error trying to write no$gba symbol file."); - } + log.Message(Logger.MessageKind.ERROR, "Error trying to write no$gba symbol file."); } } - if (EAOptions.Instance.buildTimes) { - - // Print times - - log.Output.WriteLine(); - log.Output.WriteLine("Times:"); - - foreach (KeyValuePair time in ExecTimer.Timer.SortedTimes) + if (EAOptions.BenchmarkBuildTimes) { - log.Output.WriteLine(" " + time.Value + ": " + time.Key.ToString() + " (" + ExecTimer.Timer.Counts[time.Value] + ")"); - } + // Print times + + log.Output.WriteLine(); + log.Output.WriteLine("Times:"); - // Print total time + foreach (KeyValuePair time in ExecTimer.Timer.SortedTimes) + { + log.Output.WriteLine(" " + time.Value + ": " + time.Key.ToString() + " (" + ExecTimer.Timer.Counts[time.Value] + ")"); + } - log.Output.WriteLine(); - log.Output.WriteLine("Total:"); + // Print total time - log.Output.WriteLine(" " + ExecTimer.Timer.TotalTime.ToString()); + log.Output.WriteLine(); + log.Output.WriteLine("Total:"); + log.Output.WriteLine(" " + ExecTimer.Timer.TotalTime.ToString()); } inStream.Close(); @@ -318,8 +402,6 @@ static int Main(string[] args) errorStream.Close(); return success ? EXIT_SUCCESS : EXIT_FAILURE; - } } } - diff --git a/ColorzCore/Properties/AssemblyInfo.cs b/ColorzCore/Properties/AssemblyInfo.cs deleted file mode 100644 index 4694981..0000000 --- a/ColorzCore/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("ColorzCore")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("ColorzCore")] -[assembly: AssemblyCopyright("Copyright © 2017")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("b98f7ccf-9caa-406e-88d7-2040fa99f631")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.0.2.*")] -[assembly: AssemblyFileVersion("0.0.1.0")] diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index c328067..39699a8 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -1,11 +1,8 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Collections; using ColorzCore.Parser.AST; using ColorzCore.DataTypes; -using System.Collections; +using ColorzCore.Interpreter; namespace ColorzCore.Raws { @@ -27,16 +24,18 @@ public AtomicParam(string name, int position, int length, bool isPointer) pointer = isPointer; } + // Precondition: input is an IAtomNode public void Set(byte[] data, IParamNode input) { - Set(data, input.AsAtom().FromJust.CoerceInt()); + Set(data, (input as IAtomNode)!.CoerceInt()); } public void Set(byte[] data, int value) { if (pointer && value != 0) - value |= EAOptions.Instance.romOffset; - //TODO: Perhaps additional EAOption to add for ROM offsets that aren't address spaces, e.g. program being loaded at 0x100? + { + value = EAInterpreter.ConvertToAddress(value); + } data.SetBits(Position, Length, value); } diff --git a/ColorzCore/Raws/IRawParam.cs b/ColorzCore/Raws/IRawParam.cs index a9fdaf2..9198a72 100644 --- a/ColorzCore/Raws/IRawParam.cs +++ b/ColorzCore/Raws/IRawParam.cs @@ -8,7 +8,7 @@ namespace ColorzCore.Raws { - interface IRawParam + public interface IRawParam { string Name { get; } int Position { get; } //Length and position in bits diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index d8069d0..c998bd7 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -10,23 +10,33 @@ namespace ColorzCore.Raws { - class Raw + public class Raw { public string Name { get; } public int Alignment { get; } public HashSet Game { get; } + // symbolic helper + public static HashSet AnyGame { get; } = new HashSet(); + private readonly IList parameters; private readonly bool repeatable; private readonly int unitSize; private readonly byte[] baseUnit; - private readonly byte[] endUnit; // Note: nullable + private readonly byte[]? endUnit; // TODO: fixed mask? - public Raw(string name, int length, short code, int offsetMod, HashSet game, IList varParams, - IList fixedParams, Maybe terminatingList, bool repeatable) + public struct FixedParam + { + public int position; + public int size; + public int value; + } + + public Raw(string name, int length, short code, int offsetMod, HashSet game, IList varParams, + IList? fixedParams, int? terminatingList, bool repeatable) { Name = name; Game = game; @@ -34,41 +44,49 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g this.parameters = varParams; this.repeatable = repeatable; - + unitSize = length; baseUnit = new byte[(unitSize + 7) / 8]; // Build base unit if (code != 0) + { baseUnit.SetBits(0, 16, code); + } - foreach (var fp in fixedParams) - baseUnit.SetBits(fp.position, fp.size, fp.value); + if (fixedParams != null) + { + foreach (var fp in fixedParams) + baseUnit.SetBits(fp.position, fp.size, fp.value); + } // Build end unit, if needed - if (!terminatingList.IsNothing) + if (terminatingList is int terminator) { - int terminator = terminatingList.FromJust; - if (parameters.Count == 0) return; - + endUnit = (byte[])baseUnit.Clone(); endUnit.SetBits(parameters[0].Position, parameters[0].Length, terminator); // force repeatable to be true if this is terminating list - this.repeatable = true; + this.repeatable = true; } } - public int UnitCount(int paramCount) + public Raw(string name, int length, short code, int offsetMod, IList varParams, bool repeatable) + : this(name, length, code, offsetMod, AnyGame, varParams, null, null, repeatable) + { + } + + public int UnitCount(int paramCount) { if (parameters.Count == 0) return 1; - return paramCount / parameters.Count; + return paramCount / parameters.Count; } public int LengthBits(int paramCount) @@ -77,7 +95,7 @@ public int LengthBits(int paramCount) if (endUnit != null) count++; - + return count * unitSize; } @@ -98,18 +116,18 @@ public bool Fits(IList arguments) if (arguments.Count != parameters.Count * unitCount) return false; - for (int i = 0; i < unitCount; ++i) - { - for (int j = 0; j < parameters.Count; ++j) - { + for (int i = 0; i < unitCount; ++i) + { + for (int j = 0; j < parameters.Count; ++j) + { if (!parameters[j].Fits(arguments[i * parameters.Count + j])) - return false; - } + return false; + } } return true; } - + /* Precondition: params fits the shape of this raw's params. */ public byte[] GetBytes(IList arguments) { @@ -117,29 +135,41 @@ public byte[] GetBytes(IList arguments) var count = UnitCount(arguments.Count); - for (int i = 0; i < count; ++i) - { - var unit = (byte[])baseUnit.Clone(); - + for (int i = 0; i < count; ++i) + { + var unit = (byte[])baseUnit.Clone(); + for (int j = 0; j < parameters.Count; ++j) { parameters[j].Set(unit, arguments[i * parameters.Count + j]); } - result.SetBits(i * unitSize, unitSize, unit); + result.SetBits(i * unitSize, unitSize, unit); } if (endUnit != null) result.SetBits(count * unitSize, unitSize, endUnit); return result; - } - - public struct FixedParam + } + + public string ToPrettyString() { - public int position; - public int size; - public int value; + StringBuilder sb = new StringBuilder(); + + sb.Append(Name); + + foreach (IRawParam param in parameters) + { + sb.Append(' ').Append((param is ListParam) ? $"[{param.Name}...]" : param.Name); + } + + if (repeatable) + { + sb.Append("..."); + } + + return sb.ToString(); } } } diff --git a/ColorzCore/Raws/RawReader.cs b/ColorzCore/Raws/RawReader.cs index e09915f..f646bd9 100644 --- a/ColorzCore/Raws/RawReader.cs +++ b/ColorzCore/Raws/RawReader.cs @@ -83,7 +83,7 @@ public static IEnumerable ParseAllRaws(FileStream fs) { while (!reader.EndOfStream) { - Raw raw = null; + Raw? raw = null; try { @@ -103,7 +103,7 @@ private static Raw ParseRaw(FileLineReader source) { // Since the writer of the raws is expected to know what they're doing, I'm going to be a lot more lax with error messages and graceful failure. - string rawLine; + string? rawLine; do { @@ -180,7 +180,7 @@ private static Raw ParseRaw(FileLineReader source) if (!char.IsWhiteSpace((char)next)) break; - string line = source.ReadLine(); + string? line = source.ReadLine(); if (string.IsNullOrEmpty(line) || line.Trim().Length == 0) continue; @@ -215,25 +215,25 @@ private static Raw ParseRaw(FileLineReader source) ? flags[FLAG_ALIGNMENT].Values.GetLeft[0].ToInt() : 4; - Maybe listTerminator = flags.ContainsKey(FLAG_LIST_TERMINATOR) - ? (Maybe)new Just(flags[FLAG_LIST_TERMINATOR].Values.GetLeft[0].ToInt()) - : new Nothing(); + int? listTerminator = flags.ContainsKey(FLAG_LIST_TERMINATOR) + ? flags[FLAG_LIST_TERMINATOR].Values.GetLeft[0].ToInt() + : null; - if (!listTerminator.IsNothing && code != 0) + if (listTerminator != null && code != 0) { throw new RawParseException("TerminatingList with code nonzero.", source.FileName, lineNumber); } bool isRepeatable = flags.ContainsKey(FLAG_REPEATABLE); - if ((isRepeatable || !listTerminator.IsNothing) && (parameters.Count > 1) && fixedParams.Count > 0) + if ((isRepeatable || listTerminator != null) && (parameters.Count > 1) && fixedParams.Count > 0) { throw new RawParseException("Repeatable or terminatingList code with multiple parameters or fixed parameters.", source.FileName, lineNumber); } // HACK: support terminating lists that have a code size of 0 (ugh) - if (!listTerminator.IsNothing) + if (listTerminator != null) { foreach (var param in parameters) size = Math.Max(size, param.Position + param.Length); @@ -323,7 +323,7 @@ private static Dictionary ParseFlags(string flagStr) var withoutDash = flag.Substring(1); - var name = string.Empty; + string name; var value = new Flag(); if (withoutDash.Contains(':')) @@ -348,7 +348,7 @@ private static Dictionary ParseFlags(string flagStr) name = withoutDash; } - if (FLAG_ALIAS_MAP.TryGetValue(name, out string realName)) + if (FLAG_ALIAS_MAP.TryGetValue(name, out string? realName)) { name = realName; } @@ -365,7 +365,7 @@ private class FileLineReader : IDisposable // Helper wrapper class for reading lines from file while counting them // That way we can print line number in error messages - private StreamReader reader; + private readonly StreamReader reader; public string FileName { get; } public int LineNumber { get; private set; } @@ -380,12 +380,12 @@ public FileLineReader(FileStream input) LineNumber = 0; } - public string ReadLine() + public string? ReadLine() { - string line = reader.ReadLine(); + string? line = reader.ReadLine(); if (line != null) - LineNumber = LineNumber + 1; + LineNumber++; return line; } diff --git a/ColorzCore/bin/Debug/ColorzCore.exe.config b/ColorzCore/bin/Debug/ColorzCore.exe.config deleted file mode 100644 index d740e88..0000000 --- a/ColorzCore/bin/Debug/ColorzCore.exe.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ColorzCore/bin/Release/ColorzCore.exe b/ColorzCore/bin/Release/ColorzCore.exe deleted file mode 100644 index 6f31e67..0000000 Binary files a/ColorzCore/bin/Release/ColorzCore.exe and /dev/null differ diff --git a/Tests/.gitignore b/Tests/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/Tests/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/Tests/README.md b/Tests/README.md new file mode 100644 index 0000000..1a8705c --- /dev/null +++ b/Tests/README.md @@ -0,0 +1,3 @@ +Run using `python run_tests.py path/to/ColorzCore.exe`. + +Tests are incomplete. Do not assume that because all tests are passing the program works fine. diff --git a/Tests/directives.py b/Tests/directives.py new file mode 100644 index 0000000..c2d3d98 --- /dev/null +++ b/Tests/directives.py @@ -0,0 +1,102 @@ +from ea_test import EATest as T + + +TESTS = [ + # ===================== + # = #define Directive = + # ===================== + + # '#define' traditional nominal behavior + T("Define Basic object-like", + '#define Value 0xFA \n ORG 0 ; BYTE Value', + b"\xFA"), + + T("Define Basic function-like", + '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', + b"\xFC"), + + # '#define' a second time overrides the first definition + # NOTE: this is probably not something that we want to freeze + # T("Define override", + # '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', + # b"\x02"), + + # '#define' using a vector as argument (extra commas) + T("Define vector argument", + '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', + b"\x01"), + + # '#define ... "..."' with escaped newlines inside string + T("Multi-line string define", + '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n' + + 'ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', + b"\xCC\x00\x00\x00\xBB\x00\xAA"), + + T("Define eager expansion", + "#define Value 1 \n #define OtherValue Value \n #undef Value \n #define Value 2 \n ORG 0 ; BYTE OtherValue", + b'\x01'), + + # '#define ...' multi-token without quotes + T("Multi-token define 1", + '#define Value (1 + 2) \n ORG 0 ; BYTE Value', + b"\x03"), + + T("Multi-token define 2", + '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', + b"\x03"), + + T("Multi-token define 2", + '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', + b"\x01\x03\x02"), + + # Those would fail on + T("Define uinintuitive atomic 1", + '#define Value 1 + 2 \n ORG 0 ; BYTE Value * 2', + b"\x05"), # 1 * 2 * 2 = 5 + + T("Define uinintuitive atomic 1", + '#define Value "1 + 2" \n ORG 0 ; BYTE Value * 2', + b"\x05"), # 1 * 2 * 2 = 5 + + # '#define MyMacro MyMacro' (MyMacro shouldn't expand) + T("Non-productive macros 1", + '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', + b'\x01'), + + T("Non-productive macros 2", + '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', + b'\x01'), + + T("Non-productive macros 3", + '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', + b'\x01'), + + # ==================== + # = #undef Directive = + # ==================== + + T("Undef 1", + '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', + None), + + T("Undef 2", + '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', + b"\x01"), + + T("Undef multiple", + '#define ValueA \n #define ValueB \n #undef ValueA ValueB \n #ifndef ValueA \n #ifndef ValueB \n' + + 'BYTE 1 \n #endif \n #endif', + b'\x01'), + + # ======================== + # = #if[n]def Directives = + # ======================== + + # '#ifdef' + T("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + + # '#ifndef' + T("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), + + # TODO: #if, #include, #incbin, #incext, #inctext, #pool +] diff --git a/Tests/ea_test.py b/Tests/ea_test.py new file mode 100644 index 0000000..9ec480a --- /dev/null +++ b/Tests/ea_test.py @@ -0,0 +1,84 @@ +import os, subprocess, tempfile + + +class EATestConfig: + command : list[str] + extra_params : list[str] + + def __init__(self, command : str, extra_params : str | None) -> None: + self.command = command.split() + self.extra_params = extra_params.split() if extra_params is not None else [] + + +class EATest: + name : str + script : str + expected : bytes | None + + def __init__(self, name : str, script : str, expected : bytes | None) -> None: + self.name = name + self.script = script + self.expected = expected + + def run_test(self, config : EATestConfig) -> bool: + success = False + + # TODO: this could be better than just "success/failure" + # Failure here can be two causes: result doesn't match OR program crashed. + + with tempfile.NamedTemporaryFile(delete = False) as f: + f.close() + + completed = subprocess.run(config.command + ["A", "FE6", f"-output:{f.name}"] + config.extra_params, + text = True, input = self.script, stdout = subprocess.DEVNULL, stderr = subprocess.PIPE) + + if self.expected is None: + # success on error + success = completed.returncode != 0 and "Errors occurred; no changes written." in completed.stderr + + else: + # success on resulting bytes matching + with open(f.name, 'rb') as f2: + result_bytes = f2.read() + + success = result_bytes == self.expected + + os.remove(f.name) + + return success + + +HEADER = '\033[95m' +CC_OKBLUE = '\033[94m' +CC_OKCYAN = '\033[96m' +CC_OKGREEN = '\033[92m' +CC_WARNING = '\033[93m' +CC_FAIL = '\033[91m' +CC_ENDC = '\033[0m' +CC_BOLD = '\033[1m' +CC_UNDERLINE = '\033[4m' + + +SUCCESS_MESSAGE = f"{CC_OKBLUE}SUCCESS{CC_ENDC}" +FAILURE_MESSAGE = f"{CC_FAIL}FAILURE{CC_ENDC}" + + +def run_tests(config : EATestConfig, test_cases : list[EATest]) -> None: + success_count = 0 + test_count = len(test_cases) + + for i, test_case in enumerate(test_cases): + success = test_case.run_test(config) + + message = SUCCESS_MESSAGE if success else FAILURE_MESSAGE + print(f"[{i + 1}/{test_count}] {test_case.name}: {message}") + + if success: + success_count = success_count + 1 + + if success_count == test_count: + print(f"{success_count}/{test_count} tests passed {SUCCESS_MESSAGE}") + + else: + print(f"{success_count}/{test_count} tests passed {FAILURE_MESSAGE}") + diff --git a/Tests/expressions.py b/Tests/expressions.py new file mode 100644 index 0000000..5b740ca --- /dev/null +++ b/Tests/expressions.py @@ -0,0 +1,90 @@ +from ea_test import EATest as T + +def byte(expr): + return f"ORG 0 ; BYTE {expr} ;" + +TESTS = [ + T("Operator '+'", byte('1 + 2'), b'\x03'), + + T("Operator '-' 1", byte('2 - 1'), b'\x01'), + T("Operator '-' 2", byte('1 - 2'), b'\xFF'), + + T("Operator '*'", byte('3 * 2'), b'\x06'), + + T("Operator '/' 1", byte('6 / 2'), b'\x03'), + T("Operator '/' 2", byte('5 / 2'), b'\x02'), # +2 (round towards zero) + T("Operator '/' 3", byte('(-5) / 2'), b'\xFE'), # -2 (round towards zero) + + T("Operator '%' 1", byte('5 % 2'), b'\x01'), # +1 + T("Operator '%' 2", byte('(-5) % 2'), b'\xFF'), # -1 + + T("Operator '<<'", byte('3 << 2'), b'\x0C'), # 12 + T("Operator '>>'", byte('3 >> 1'), b'\x01'), # 12 + T("Operator '>>>'", byte('0x80000000 >>> 25'), b'\xC0'), # 0b11000000 + + T("Operator '<' 1", byte('1 < 2'), b'\x01'), + T("Operator '<' 2", byte('2 < 1'), b'\x00'), + T("Operator '<' 3", byte('2 < 2'), b'\x00'), + + T("Operator '<=' 1", byte('1 <= 2'), b'\x01'), + T("Operator '<=' 2", byte('2 <= 1'), b'\x00'), + T("Operator '<=' 3", byte('2 <= 2'), b'\x01'), + + T("Operator '==' 1", byte('1 == 2'), b'\x00'), + T("Operator '==' 2", byte('2 == 1'), b'\x00'), + T("Operator '==' 3", byte('2 == 2'), b'\x01'), + + T("Operator '!=' 1", byte('1 != 2'), b'\x01'), + T("Operator '!=' 2", byte('2 != 1'), b'\x01'), + T("Operator '!=' 3", byte('2 != 2'), b'\x00'), + + T("Operator '>=' 1", byte('1 >= 2'), b'\x00'), + T("Operator '>=' 2", byte('2 >= 1'), b'\x01'), + T("Operator '>=' 3", byte('2 >= 2'), b'\x01'), + + T("Operator '>' 1", byte('1 > 2'), b'\x00'), + T("Operator '>' 2", byte('2 > 1'), b'\x01'), + T("Operator '>' 3", byte('2 > 2'), b'\x00'), + + T("Operator '&' 1", byte('3 & 6'), b'\x02'), + T("Operator '&' 2", byte('1 & 6'), b'\x00'), + + T("Operator '|' 1", byte('1 | 12'), b'\x0D'), # 0b1101 + T("Operator '|' 2", byte('1 | 1'), b'\x01'), + + T("Operator '^' 1", byte('3 ^ 6'), b'\x05'), + T("Operator '^' 2", byte('1 ^ 6'), b'\x07'), + + T("Operator '&&' 1", byte('0 && 1'), b'\x00'), + T("Operator '&&' 2", byte('1 && 1'), b'\x01'), + T("Operator '&&' 3", byte('1 && 10'), b'\x0A'), + + T("Operator '||' 1", byte('0 || 1'), b'\x01'), + T("Operator '||' 2", byte('1 || 1'), b'\x01'), + T("Operator '||' 3", byte('1 || 10'), b'\x01'), + T("Operator '||' 4", byte('0 || 10'), b'\x0A'), + T("Operator '||' 5", byte('8 || 1'), b'\x08'), + + T("Operator '??' 1", f'A := 0 ;' + byte('(A || 1) ?? 0'), b"\x01"), + T("Operator '??' 2", byte('(A || 1) ?? 0'), b"\x00"), + + T("Operator unary '-' 1", byte('-1'), b'\xFF'), + T("Operator unary '-' 2", byte('-(1 + 2)'), b'\xFD'), + + T("Operator unary '~' 1", byte('~0'), b'\xFF'), + T("Operator unary '~' 2", byte('~3'), b'\xFC'), + T("Operator unary '~' 3", byte('~(-1)'), b'\x00'), + + T("Operator unary '!' 1", byte('!76'), b'\x00'), + T("Operator unary '!' 2", byte('!0'), b'\x01'), + T("Operator unary '!' 3", byte('!!7'), b'\x01'), + + T("Precedence 1 ('+', '*')", byte('1 + 2 * 3'), b'\x07'), # +7 + T("Precedence 2 ('-', '*')", byte('1 - 2 * 3'), b'\xFB'), # -5 + T("Precedence 3 ('+', '/')", byte('4 + 6 / 2'), b'\x07'), # +7 (not +5) + T("Precedence 4 ('+', '%')", byte('5 + 5 % 2'), b'\x06'), # +6 (not +0) + T("Precedence 5 ('<<', '+')", byte('2 << 1 + 5'), bytes((128,))), # not 9 + T("Precedence 6 ('>>', '+')", byte('0xFF >> 1 + 5'), b'\x03'), + T("Precedence 7 ('>>>', '+')", byte('0x80000000 >>> 20 + 5'), b'\xC0'), + # TODO: more +] diff --git a/Tests/run_tests.py b/Tests/run_tests.py new file mode 100644 index 0000000..19b375a --- /dev/null +++ b/Tests/run_tests.py @@ -0,0 +1,41 @@ +import sys +from ea_test import EATestConfig as Config, EATest as T, run_tests + +BASIC_TESTS = [ + T("Basic", "ORG 0 ; BYTE 1", b"\x01"), + T("Addition", "ORG 0 ; BYTE 1 + 2", b"\x03"), + T("Precedence 1", "ORG 0 ; BYTE 1 + 2 * 10", b"\x15"), + + # POIN + T("POIN Offset", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), + T("POIN NULL", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), + T("POIN Address", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), + T("POIN RAM", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), +] + + +import statements, symbols, directives, expressions + +ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + symbols.TESTS + directives.TESTS + expressions.TESTS + +def main(args): + import argparse + + arg_parse = argparse.ArgumentParser() + + arg_parse.add_argument("command") + arg_parse.add_argument("--extra-params") + + args = arg_parse.parse_args(args[1:]) + + command : str = args.command + extra_params : str = args.extra_params + + test_cases = ALL_TEST_CASES + + config = Config(command, extra_params) + run_tests(config, test_cases) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/Tests/statements.py b/Tests/statements.py new file mode 100644 index 0000000..083a05d --- /dev/null +++ b/Tests/statements.py @@ -0,0 +1,170 @@ +from ea_test import EATest as T + + +TESTS = [ + # ================= + # = ORG Statement = + # ================= + + # Nominal behavior + T("ORG Basic", + "ORG 1 ; BYTE 1 ; ORG 10 ; BYTE 10", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Also works backwards + T("ORG Backwards", + "ORG 10 ; BYTE 10 ; ORG 1 ; BYTE 1", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Addresses are Offsets + T("ORG Addresses", + "ORG 0x08000001 ; BYTE 1 ; ORG 0x0800000A ; BYTE 10", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Error on offset too big + T("ORG Overflow", + "ORG 0x10000000 ; BYTE 1", + None), + + # Error on offset too small/negative + T("ORG Underflow", + "ORG -1 ; BYTE 1", + None), + + # ======================= + # = PUSH/POP Statements = + # ======================= + + # Nominal behavior + T("PUSH POP Basic", + "ORG 4 ; PUSH ; ORG 1 ; POP ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("PUSH POP Override", + "ORG 0 ; PUSH ; BYTE 0xAA ; POP ; BYTE 0xBB", + b"\xBB"), + + T("POP Naked", + "ORG 0 ; BYTE 0 ; POP", + None), + + # =================== + # = ALIGN Statement = + # =================== + + T("ALIGN Basic", + "ORG 1 ; ALIGN 4 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("ALIGN Aligned", + "ORG 4 ; ALIGN 4 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("ALIGN Zero", + "ORG 1 ; ALIGN 0 ; BYTE CURRENTOFFSET", + None), + + T("ALIGN Negative", + "ORG 1 ; ALIGN -1 ; BYTE CURRENTOFFSET", + None), + + T("ALIGN Offset", + "ORG 2 ; ALIGN 4 1 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x00\x05"), + + T("ALIGN Offset Aligned", + "ORG 1 ; ALIGN 4 1 ; BYTE CURRENTOFFSET", + b"\x00\x01"), + + # ================== + # = FILL Statement = + # ================== + + T("FILL Basic", + "ORG 0 ; FILL 0x10", + b"\x00" * 0x10), + + T("FILL Value", + "ORG 4 ; FILL 0x10 0xFF", + b"\x00\x00\x00\x00" + b"\xFF" * 0x10), + + T("FILL Zero", + "ORG 0 ; FILL 0", + None), + + T("FILL Negative", + "ORG -1 ; FILL 0", + None), + + # ==================== + # = ASSERT Statement = + # ==================== + + T("ASSERT Traditional", + "ASSERT 0", + b""), + + T("ASSERT Traditional Failure", + "ASSERT -1", + None), + + T("ASSERT Conditional", + "ASSERT 1 > 0", + b""), + + T("ASSERT Conditional Failure", + "ASSERT 1 < 0", + None), + + T("ASSERT Traditional Expression Failure", + "ASSERT 1 - 2", + None), + + # ==================== + # = STRING Statement = + # ==================== + # incomplete + + T("STRING Basic", + "ORG 0 ; STRING \"Hello World\"", + b"Hello World"), + + # ==================== + # = BASE64 Statement = + # ==================== + + T("BASE64", + " BASE64 \"RXZlbnQgQXNzZW1ibGVy\"", + b"Event Assembler"), + + # ===================== + # = PROTECT Statement = + # ===================== + + T("PROTECT Basic", + "PROTECT 0 4 ; ORG 0 ; BYTE 1", + None), + + T("PROTECT Edge 1", + "PROTECT 0 4 ; ORG 3 ; BYTE 1", + None), + + T("PROTECT Edge 2", + "PROTECT 0 4 ; ORG 4 ; BYTE 1", + b"\x00\x00\x00\x00\x01"), + + T("PROTECT Late", + "ORG 0 ; BYTE 1 ; PROTECT 0 4", + b"\x01"), + + # default PROTECT end is start + 4 + T("PROTECT Default range 1", + "PROTECT 0 ; ORG 4 ; BYTE 1", + b"\x00\x00\x00\x00\x01"), + + # default PROTECT end is start + 4 + T("PROTECT Default range 2", + "PROTECT 0 ; ORG 3 ; BYTE 1", + None), + +] diff --git a/Tests/symbols.py b/Tests/symbols.py new file mode 100644 index 0000000..13c99b1 --- /dev/null +++ b/Tests/symbols.py @@ -0,0 +1,48 @@ +from ea_test import EATest as T + + +TESTS = [ + T("Label Basic", + "ORG 4 ; MyLabel: ; ORG 0 ; BYTE MyLabel", + b'\x04'), + + T("Label None", + "ORG 0 ; BYTE MyLabel", + None), + + T("Label Address", + "ORG 4 ; MyLabel: ; ORG 0 ; WORD MyLabel", + b'\x04\x00\x00\x08'), + + T("Label POIN", + "ORG 4 ; MyLabel: ; ORG 0 ; POIN MyLabel", + b'\x04\x00\x00\x08'), + + T("Label Forward", + "ORG 0 ; WORD MyLabel ; MyLabel:", + b'\x04\x00\x00\x08'), + + T("Symbol Basic", + "MySymbol := 0xBEEF ; ORG 0 ; SHORT MySymbol", + b'\xEF\xBE'), + + T("Symbol Reference Forward", + "ORG 0 ; SHORT MySymbol ; MySymbol := 0xBEEF", + b'\xEF\xBE'), + + T("Symbol Evaluate Forward", + "MySymbol := MyLabel + 0xA0 ; ORG 0 ; BYTE MySymbol ; MyLabel:", + b'\xA1'), + + T("Scope Basic", + "ORG 0 ; { MyLabel: BYTE MyLabel + 1 ; }", + b'\x01'), + + T("Scope Failure", + "ORG 0 ; { MyLabel: BYTE 0 ; } BYTE MyLabel", + None), + + T("Scope Up", + "ORG 0 ; MyLabel: { BYTE MyLabel + 1 ; }", + b'\x01'), +]