From c5fc37cc59d952eb8948428fb009644179cdf863 Mon Sep 17 00:00:00 2001 From: JocelynSachs Date: Tue, 25 Aug 2015 17:03:03 +0100 Subject: [PATCH 1/4] Optimised ModuleDecl.GetDecl Added caching of indirectly accessible decls to speed up semanting on projects with large numbers of files. --- modules/trans/decl.monkey | 261 +++++++++++++++++++++++++++++++++++++- 1 file changed, 259 insertions(+), 2 deletions(-) diff --git a/modules/trans/decl.monkey b/modules/trans/decl.monkey index f75d352e..3d6cc488 100644 --- a/modules/trans/decl.monkey +++ b/modules/trans/decl.monkey @@ -1261,8 +1261,26 @@ End Const MODULE_STRICT=1 Const MODULE_SEMANTALL=2 +Class DeclMDeclPair + Field tdecl:Object + Field mdecl:ModuleDecl +End + +Class SynonymList + Field list:List = New List +End + Class ModuleDecl Extends ScopeDecl + Global master_uid:Int = 0 + + Field uid:Int = 0 + Field indirectMapsPrepped:Bool = False + Field indirectDeclMapPublicOnly:= New StringMap + Field indirectDeclMapPublicPrivate:= New StringMap + Field accessibleModuleScopesPublic:= New IntMap + Field accessibleModuleScopesPublicPrivate:= New IntMap + Field modpath$,rmodpath$,filepath$ Field imported:=New StringMap 'Maps filepath to modules Field pubImported:=New StringMap 'Ditto for publicly imported modules @@ -1274,6 +1292,9 @@ Class ModuleDecl Extends ScopeDecl Method New( ident$,attrs,munged$,modpath$,filepath$,app:AppDecl ) + Self.uid = master_uid + master_uid += 1 + Self.ident=ident Self.attrs=attrs Self.munged=munged @@ -1296,7 +1317,243 @@ Class ModuleDecl Extends ScopeDecl Return (attrs & MODULE_STRICT)<>0 End - Method GetDecl:Object( ident$ ) + Method GetDeclFromImports:Object(ident:String, map:StringMap, decl:Object) + + Local candidates:SynonymList = map.Get(ident) + If Not candidates + ScopeDecl.GetDecl_Fail += 1 + Return decl + EndIf + ScopeDecl.GetDecl_Success += 1 + Local n:= candidates.list.FirstNode + Local pair:DeclMDeclPair + While n + pair = n.Value + Local mdecl:ModuleDecl = pair.mdecl + Local tdecl:Object = pair.tdecl + + 'AC: Handle aliases here (it's normally done inside Super.GetDecl but we don't call that.) + Local adecl:= AliasDecl(tdecl) + If adecl + tdecl = Null + If adecl.CheckAccess() tdecl = adecl.decl + Endif + + 'ignore private decls + Local ddecl:= Decl(tdecl) + If ddecl And Not ddecl.CheckAccess() tdecl=Null + + 'ignore funclists with no public funcs + Local flist:=FuncDeclList( tdecl ) + If flist + Local pub:=False + For Local fdecl:=Eachin flist + If Not fdecl.CheckAccess() Continue + pub=True + Exit + Next + If Not pub tdecl=Null + EndIf + + If tdecl And tdecl <> decl + If mdecl=Self Return tdecl + If decl + 'AC: Easier to let the slow version handle this error: + GetDecl_Slow(ident) + 'AC: We should definitely not get back from that call. If we do, this optimised process is broken! + Err "Uh oh! Duplicate identifier '" + ident + "' found via GelDecl but not via GetDecl_Slow!" + Endif + decl=tdecl + Endif + n = n.NextNode + Wend + + Return decl + + End + + Method BuildIndirectDeclMaps() + + indirectDeclMapPublicOnly = New StringMap + indirectDeclMapPublicPrivate = New StringMap + accessibleModuleScopesPublic = New IntMap + accessibleModuleScopesPublicPrivate = New IntMap + + indirectMapsPrepped = True + Local todo:= New List + Local accListPublic:List = New List + Local accListPublicPrivate:List = New List + + 'AC: First create the list of accessible modules publicly available via this module scope. + + todo.AddLast Self + accessibleModuleScopesPublic.Set(uid, Self) + accListPublic.AddLast(Self) + accListPublicPrivate.AddLast(Self) + + While Not todo.IsEmpty() + + Local mdecl:ModuleDecl = todo.RemoveLast() + + Local imps:= mdecl.pubImported + + For Local mdecl2:= EachIn imps.Values() + If Not accessibleModuleScopesPublic.Contains(mdecl2.uid) + todo.AddLast mdecl2 + accessibleModuleScopesPublic.Set(mdecl2.uid, mdecl2) + accListPublic.AddLast(mdecl2) + accListPublicPrivate.AddLast(mdecl2) + Endif + Next + Wend + + 'AC: Now create an expanded list of modules available privately to this module scope. + + Local extraPrivates:Bool = False + accessibleModuleScopesPublicPrivate.Clone(accessibleModuleScopesPublic) + For Local mdecl2:= EachIn imported.Values() + If Not accessibleModuleScopesPublicPrivate.Contains(mdecl2.uid) + todo.AddLast mdecl2 + accessibleModuleScopesPublicPrivate.Set(mdecl2.uid, mdecl2) + accListPublicPrivate.AddLast(mdecl2) + extraPrivates = True + Endif + Next + + While Not todo.IsEmpty() + + Local mdecl:ModuleDecl = todo.RemoveLast() + + Local imps:= mdecl.pubImported + + For Local mdecl2:= EachIn imps.Values() + If Not accessibleModuleScopesPublicPrivate.Contains(mdecl2.uid) + todo.AddLast mdecl2 + accessibleModuleScopesPublicPrivate.Set(mdecl2.uid, mdecl2) + accListPublicPrivate.AddLast(mdecl2) + extraPrivates = True + Endif + Next + Wend + + 'AC: Now build decl maps from those accessibility lists + BuildIndirectDeclMap(indirectDeclMapPublicOnly, accListPublic) + If extraPrivates + BuildIndirectDeclMap(indirectDeclMapPublicPrivate, accListPublicPrivate) + Else + indirectDeclMapPublicPrivate = indirectDeclMapPublicOnly + EndIf + + End + + Method AddIndirectDecl(idm:StringMap, ident:String, decl:Object, mdecl:ModuleDecl) + Local syn:SynonymList = idm.Get(ident) + Local dmdp:DeclMDeclPair + If syn = Null + syn = New SynonymList + idm.Set(ident, syn) + EndIf + Local n:= syn.list.FirstNode + While n + dmdp = n.Value + If dmdp.tdecl = decl + Return + EndIf + n = n.NextNode + Wend + + dmdp = New DeclMDeclPair + dmdp.tdecl = decl + dmdp.mdecl = mdecl + syn.list.AddLast(dmdp) + End + + Method BuildIndirectDeclMap(idm:StringMap, acclist:List) + Local mn:= acclist.FirstNode + While mn + Local declmap:StringMap = mn.Value.declsMap + Local dn:= declmap.FirstNode + While dn + AddIndirectDecl(idm, dn.Key, dn.Value, mn.Value) + dn = dn.NextNode + Wend + mn = mn.NextNode + Wend + End + + Method GetDecl:Object(ident:String) + + If Not indirectMapsPrepped + 'Print "Building IDMs for " + Self.ident + "..." + BuildIndirectDeclMaps() + 'Print "Done." + EndIf + + If Not _env + Return GetDecl_Slow(ident) + EndIf + + Local mdecl:ModuleDecl = _env.ModuleScope() + + Local decl:Object + + If Self = mdecl + 'AC: Just check the indirect map that includes this module's public AND private imports. + 'Print "Scope match! Checking public/private map." + decl = GetDeclFromImports(ident, indirectDeclMapPublicPrivate, Null) + 'Print "Done." + Else + If accessibleModuleScopesPublic.Contains(mdecl.uid) + If Not mdecl.indirectMapsPrepped + mdecl.BuildIndirectDeclMaps() + EndIf + 'AC: The _env's own module scope is accessible from here. + 'AC: Check the indirect map that only includes this module's public imports, + ' AND the _env's own module scope's public/private map. + 'Print "Scope mismatch, but env scope accessible! Checking public map and env scope public/private map." + decl = GetDeclFromImports(ident, mdecl.indirectDeclMapPublicPrivate, GetDeclFromImports(ident, indirectDeclMapPublicOnly, Null)) + 'Print "Done." + Else + 'AC: The _env's own module scope is NOT accessible from here. + ' Just check our public imports. + 'Print "Scope mismatch, env scope inaccessible! Checking public map." + decl = GetDeclFromImports(ident, indirectDeclMapPublicOnly, Null) + 'Print "Done." + EndIf + EndIf + + #rem + + 'AC: Uncomment this to validate decls found against the original method, and debug why they're different (if they are!) + Local decl2:Object = GetDecl_Slow(ident) + + If decl <> decl2 + 'AC: Uh oh. + DebugStop + 'AC: Do it again so we can watch. + decl2 = GetDecl_Slow(ident) + DebugStop + 'AC: Build the maps again so we can watch. + BuildIndirectDeclMaps() + DebugStop + 'AC: Retrieve the decl again so we can watch. + If Self = mdecl + decl = GetDeclFromImports(ident, indirectDeclMapPublicPrivate, Null) + Else + If accessibleModuleScopesPublic.Contains(mdecl.uid) + decl = GetDeclFromImports(ident, mdecl.indirectDeclMapPublicPrivate, GetDeclFromImports(ident, indirectDeclMapPublicOnly, Null)) + Else + decl = GetDeclFromImports(ident, indirectDeclMapPublicOnly, Null) + EndIf + EndIf + EndIf + #end + + Return decl + + End + + Method GetDecl_Slow:Object(ident:String) Local todo:=New List Local done:=New StringMap @@ -1304,7 +1561,7 @@ Class ModuleDecl Extends ScopeDecl todo.AddLast Self done.Insert filepath,Self - Local decl:Object,declmod$ + Local decl:Object, declmod:String While Not todo.IsEmpty() From b9f9a455c89b539cd013f4e91b6ea517561bb7b3 Mon Sep 17 00:00:00 2001 From: JocelynSachs Date: Tue, 25 Aug 2015 17:07:22 +0100 Subject: [PATCH 2/4] Added 'Clone' method to Map Used by optimised ModuleDecl.GetDecl. --- modules/monkey/map.monkey | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/modules/monkey/map.monkey b/modules/monkey/map.monkey index 925889c4..b50cf696 100644 --- a/modules/monkey/map.monkey +++ b/modules/monkey/map.monkey @@ -141,6 +141,14 @@ Class Map Return node End + Method Clone(from:Map) + If Not from.root + root = Null + Return + EndIf + root = from.root.Copy(Null) + End + 'Deprecated - use Set Method Insert:Bool( key:K,value:V ) Return Set( key,value ) From 44377342230524dff909e0a9a7e4f15209e9423e Mon Sep 17 00:00:00 2001 From: JocelynSachs Date: Tue, 25 Aug 2015 20:33:30 +0100 Subject: [PATCH 3/4] Corrected ModuleDecl 'dirty' logic ALL caches are now dirtied when a module has decls added (since those caches could contain indirect references that are now out of date. --- modules/trans/decl.monkey | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/modules/trans/decl.monkey b/modules/trans/decl.monkey index 3d6cc488..4a9bd1e8 100644 --- a/modules/trans/decl.monkey +++ b/modules/trans/decl.monkey @@ -1273,9 +1273,10 @@ End Class ModuleDecl Extends ScopeDecl Global master_uid:Int = 0 - + Global master_im_version:Int = 1 + Field uid:Int = 0 - Field indirectMapsPrepped:Bool = False + Field indirectMapsVersion:Int = 0 Field indirectDeclMapPublicOnly:= New StringMap Field indirectDeclMapPublicPrivate:= New StringMap Field accessibleModuleScopesPublic:= New IntMap @@ -1379,7 +1380,7 @@ Class ModuleDecl Extends ScopeDecl accessibleModuleScopesPublic = New IntMap accessibleModuleScopesPublicPrivate = New IntMap - indirectMapsPrepped = True + indirectMapsVersion = master_im_version Local todo:= New List Local accListPublic:List = New List Local accListPublicPrivate:List = New List @@ -1504,7 +1505,7 @@ Class ModuleDecl Extends ScopeDecl 'Print "Done." Else If accessibleModuleScopesPublic.Contains(mdecl.uid) - If Not mdecl.indirectMapsPrepped + If mdecl.indirectMapsVersion < master_im_version mdecl.BuildIndirectDeclMaps() EndIf 'AC: The _env's own module scope is accessible from here. @@ -1703,6 +1704,15 @@ Class ModuleDecl Extends ScopeDecl attrs|=MODULE_SEMANTALL End + Method InsertDecl(decl:Decl) + Super.InsertDecl(decl) + master_im_version += 1 + End + + Method InsertDecls(decls:List) + Super.InsertDecls(decls) + master_im_version += 1 + End End Class AppDecl Extends ScopeDecl From 4e59d0ee16e75140b23db339811569d17031fe2f Mon Sep 17 00:00:00 2001 From: JocelynSachs Date: Wed, 26 Aug 2015 10:57:17 +0100 Subject: [PATCH 4/4] Compile error fixes Thanks Anthony for the heads up. --- modules/trans/decl.monkey | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/trans/decl.monkey b/modules/trans/decl.monkey index 4a9bd1e8..1da1ca8a 100644 --- a/modules/trans/decl.monkey +++ b/modules/trans/decl.monkey @@ -368,6 +368,9 @@ End Class ScopeDecl Extends Decl + Global GetDecl_Fail:Int + Global GetDecl_Success:Int + Private Field decls:=New List @@ -1484,13 +1487,14 @@ Class ModuleDecl Extends ScopeDecl Method GetDecl:Object(ident:String) - If Not indirectMapsPrepped + If indirectMapsVersion < master_im_version 'Print "Building IDMs for " + Self.ident + "..." BuildIndirectDeclMaps() 'Print "Done." EndIf If Not _env + 'AC: 'Slow' version probably quicker in this case as it's a smaller stringmap. Return GetDecl_Slow(ident) EndIf