From 1999fe25dffb919f6fbd5ffd0d7f7b53975cd172 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Tue, 30 May 2023 12:05:23 +0800 Subject: [PATCH 1/7] feature(mechanism): optimize properties query frequency --- .../tron/core/db2/core/AbstractSnapshot.java | 7 ++++ .../java/org/tron/core/db2/core/Snapshot.java | 4 +++ .../org/tron/core/db2/core/SnapshotImpl.java | 35 +++++++++++++++++++ .../tron/core/db2/core/SnapshotManager.java | 6 ++++ .../org/tron/core/db2/core/SnapshotRoot.java | 4 +++ 5 files changed, 56 insertions(+) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java b/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java index 496a0fd09ac..ba6c77d43a2 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/AbstractSnapshot.java @@ -15,6 +15,8 @@ public abstract class AbstractSnapshot implements Snapshot { protected WeakReference next; + protected boolean isOptimized; + @Override public Snapshot advance() { return new SnapshotImpl(this); @@ -34,4 +36,9 @@ public void setNext(Snapshot next) { public String getDbName() { return db.getDbName(); } + + @Override + public boolean isOptimized(){ + return isOptimized; + } } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java b/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java index e1ca149b207..75545dc29b4 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/Snapshot.java @@ -46,4 +46,8 @@ static boolean isImpl(Snapshot snapshot) { void updateSolidity(); String getDbName(); + + boolean isOptimized(); + + void reloadToMem(); } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index ae8073f668b..be732214807 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -30,6 +30,12 @@ public class SnapshotImpl extends AbstractSnapshot { } previous = snapshot; snapshot.setNext(this); + isOptimized = snapshot.isOptimized(); + if (isOptimized) { + if (root == previous) { + Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue())); + } + } } @Override @@ -40,6 +46,14 @@ public byte[] get(byte[] key) { private byte[] get(Snapshot head, byte[] key) { Snapshot snapshot = head; Value value; + // for properties optimized + if (isOptimized) { + value = db.get(Key.of(key)); + if (value !=null) { + return value.getBytes(); + } + } + while (Snapshot.isImpl(snapshot)) { if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) { return value.getBytes(); @@ -83,6 +97,20 @@ public void merge(Snapshot from) { Streams.stream(fromImpl.db).forEach(e -> db.put(e.getKey(), e.getValue())); } + public void mergeFullData(Snapshot from) { + if (from instanceof SnapshotRoot) { + return ; + } + SnapshotImpl fromImpl = (SnapshotImpl) from; + Streams.stream(fromImpl.db).forEach(e -> + { + if (db.get(e.getKey()) == null && e.getValue() != null ) { + db.put(e.getKey(), e.getValue()); + } + } + ); + } + @Override public Snapshot retreat() { return previous; @@ -177,4 +205,11 @@ public String getDbName() { public Snapshot newInstance() { return new SnapshotImpl(this); } + + @Override + public void reloadToMem() { + if (isOptimized) { + mergeFullData(previous); + } + } } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java index 85d0f3c2dd0..9b27a935757 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -233,6 +233,12 @@ public synchronized void commit() { } --activeSession; + + dbs.forEach(db -> { + if (db.getHead().isOptimized()) { + db.getHead().reloadToMem(); + } + }); } public synchronized void pop() { diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java index 709e2ae1b62..f95cf68dafe 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotRoot.java @@ -38,6 +38,7 @@ public SnapshotRoot(DB db) { if (CACHE_DBS.contains(this.db.getDbName())) { this.cache = CacheManager.allocate(CacheType.findByType(this.db.getDbName())); } + isOptimized = "properties".equalsIgnoreCase(db.getDbName()); } private boolean needOptAsset() { @@ -221,4 +222,7 @@ public String getDbName() { public Snapshot newInstance() { return new SnapshotRoot(db.newInstance()); } + + @Override + public void reloadToMem() { } } From 79ca721258bc312bd511e5753e599772e4b3b217 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Fri, 2 Jun 2023 15:40:17 +0800 Subject: [PATCH 2/7] feature(mechanism): optimize properties query frequency --- .../src/main/java/org/tron/core/db2/core/SnapshotImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index be732214807..8b6b51f2c54 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -47,12 +47,14 @@ private byte[] get(Snapshot head, byte[] key) { Snapshot snapshot = head; Value value; // for properties optimized + /* if (isOptimized) { value = db.get(Key.of(key)); if (value !=null) { return value.getBytes(); } } + */ while (Snapshot.isImpl(snapshot)) { if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) { From dced678df0ce147f3ff88682e6fc6dfdc88b1189 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Wed, 9 Aug 2023 17:43:32 +0800 Subject: [PATCH 3/7] feature(mechanism): optimize properties query frequency --- .../org/tron/core/db2/core/SnapshotImpl.java | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index 8b6b51f2c54..c27b1ac377c 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -31,10 +31,8 @@ public class SnapshotImpl extends AbstractSnapshot { previous = snapshot; snapshot.setNext(this); isOptimized = snapshot.isOptimized(); - if (isOptimized) { - if (root == previous) { - Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue())); - } + if (isOptimized && root == previous) { + Streams.stream(root.iterator()).forEach( e -> put(e.getKey(),e.getValue())); } } @@ -46,15 +44,6 @@ public byte[] get(byte[] key) { private byte[] get(Snapshot head, byte[] key) { Snapshot snapshot = head; Value value; - // for properties optimized - /* - if (isOptimized) { - value = db.get(Key.of(key)); - if (value !=null) { - return value.getBytes(); - } - } - */ while (Snapshot.isImpl(snapshot)) { if ((value = ((SnapshotImpl) snapshot).db.get(Key.of(key))) != null) { @@ -99,17 +88,16 @@ public void merge(Snapshot from) { Streams.stream(fromImpl.db).forEach(e -> db.put(e.getKey(), e.getValue())); } - public void mergeFullData(Snapshot from) { + public void mergeAhead(Snapshot from) { if (from instanceof SnapshotRoot) { return ; } SnapshotImpl fromImpl = (SnapshotImpl) from; - Streams.stream(fromImpl.db).forEach(e -> - { - if (db.get(e.getKey()) == null && e.getValue() != null ) { - db.put(e.getKey(), e.getValue()); - } - } + Streams.stream(fromImpl.db).forEach(e -> { + if (db.get(e.getKey()) == null && e.getValue() != null) { + db.put(e.getKey(), e.getValue()); + } + } ); } @@ -211,7 +199,7 @@ public Snapshot newInstance() { @Override public void reloadToMem() { if (isOptimized) { - mergeFullData(previous); + mergeAhead(previous); } } } From 3b407716dc19119998c6abb9566ccaa7417b9592 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Wed, 30 Aug 2023 15:18:06 +0800 Subject: [PATCH 4/7] feature(mechanism): optimize properties query frequency --- .../src/main/java/org/tron/core/db2/core/SnapshotImpl.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index c27b1ac377c..53cd5df168d 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -198,8 +198,6 @@ public Snapshot newInstance() { @Override public void reloadToMem() { - if (isOptimized) { mergeAhead(previous); - } } } From 2ca5129ef17e37180e9bcff927704eb28ba2e410 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Wed, 30 Aug 2023 20:27:15 +0800 Subject: [PATCH 5/7] feature(mechanism): optimize properties query frequency --- .../src/main/java/org/tron/core/db2/core/SnapshotImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java index 53cd5df168d..bc31b406b30 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotImpl.java @@ -94,7 +94,7 @@ public void mergeAhead(Snapshot from) { } SnapshotImpl fromImpl = (SnapshotImpl) from; Streams.stream(fromImpl.db).forEach(e -> { - if (db.get(e.getKey()) == null && e.getValue() != null) { + if (db.get(e.getKey()) == null) { db.put(e.getKey(), e.getValue()); } } From a8a7492f68d6e267338aac9565f0463af844f3e8 Mon Sep 17 00:00:00 2001 From: chaozhu Date: Thu, 31 Aug 2023 20:25:52 +0800 Subject: [PATCH 6/7] feature(mechanism): add unit test for properties db optimization --- .../org/tron/core/db2/SnapshotImplTest.java | 209 ++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java new file mode 100644 index 00000000000..c40aaf760e0 --- /dev/null +++ b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java @@ -0,0 +1,209 @@ +package org.tron.core.db2; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import java.io.File; +import java.lang.reflect.Constructor; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.tron.common.application.Application; +import org.tron.common.application.ApplicationFactory; +import org.tron.common.application.TronApplicationContext; +import org.tron.common.utils.FileUtil; +import org.tron.core.Constant; +import org.tron.core.config.DefaultConfig; +import org.tron.core.config.args.Args; +import org.tron.core.db2.core.Snapshot; +import org.tron.core.db2.core.SnapshotImpl; +import org.tron.core.db2.core.SnapshotManager; +import org.tron.core.db2.core.SnapshotRoot; + +public class SnapshotImplTest { + private RevokingDbWithCacheNewValueTest.TestRevokingTronStore tronDatabase; + private TronApplicationContext context; + private Application appT; + private SnapshotManager revokingDatabase; + + @Before + public void init() { + Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); + context = new TronApplicationContext(DefaultConfig.class); + appT = ApplicationFactory.create(context); + } + + @After + public void removeDb() { + Args.clearParam(); + context.destroy(); + FileUtil.deleteDir(new File("output_revokingStore_test")); + } + + /** + * linklist is: from -> root + * root:key1=>value1, key2=>value2 + * from:key3=>value3, key4=>value4 + * after construct, getSnapshotImplIns(root); + * from: key1=>value1, key2=>value2, key3=>value3, key4=>value4 + * from: get key1 or key2, traverse 0 times + */ + @Test + public void testMergeRoot() { + tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( + "testSnapshotRoot-testMerge"); + revokingDatabase = context.getBean(SnapshotManager.class); + revokingDatabase.enable(); + revokingDatabase.add(tronDatabase.getRevokingDB()); + + // linklist is: from -> root + SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); + root.setOptimized(true); + + root.put("key1".getBytes(), "value1".getBytes()); + root.put("key2".getBytes(), "value2".getBytes()); + SnapshotImpl from = getSnapshotImplIns(root); + from.put("key3".getBytes(), "value3".getBytes()); + from.put("key4".getBytes(), "value4".getBytes()); + + byte[] s1 = from.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s1)); + byte[] s2 = from.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s2)); + } + + /** + * linklist is: from2 -> from -> root + * root: + * from:key1=>value1, key2=>value2 + * from2:key3=>value3,key4=>value4 + * before merge: from2.mergeAhead(from); + * from2: get key1 or key2, traverse 1 times + * after merge + * from2:key1=>value1, key2=>value2, value3=>value3,key4=>value4 + * from2: get key1 or key2, traverse 0 times + * + */ + @Test + public void testMergeAhead() { + tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( + "testSnapshotRoot-testMerge"); + revokingDatabase = context.getBean(SnapshotManager.class); + revokingDatabase.enable(); + revokingDatabase.add(tronDatabase.getRevokingDB()); + + // linklist is: from2 -> from -> root + SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); + SnapshotImpl from = getSnapshotImplIns(root); + from.put("key1".getBytes(), "value1".getBytes()); + from.put("key2".getBytes(), "value2".getBytes()); + + SnapshotImpl from2 = getSnapshotImplIns(from); + from2.put("key3".getBytes(), "value3".getBytes()); + from2.put("key4".getBytes(), "value4".getBytes()); + + /* + // before merge get data in from is success，traverse 0 times + byte[] s1 = from.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s1)); + byte[] s2 = from.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s2)); + // before merge get data in from2 is success， traverse 0 times + byte[] s3 = from2.get("key3".getBytes()); + assertEquals(new String("value3".getBytes()), new String(s3)); + byte[] s4 = from2.get("key4".getBytes()); + assertEquals(new String("value4".getBytes()), new String(s4)); + */ + + // before merge from2 get data is success， traverse 1 times + byte[] s11 = from2.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s11)); + byte[] s12 = from2.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s12)); + // this can not get key3 and key4 + assertNull(from.get("key3".getBytes())); + assertNull(from.get("key4".getBytes())); + + // do mergeAhead + from2.mergeAhead(from); + /* + // after merge get data in from is success， traverse 0 times + s1 = from.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s1)); + s2 = from.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s2)); + + // after merge get data in from2 is success， traverse 0 times + s3 = from2.get("key3".getBytes()); + assertEquals(new String("value3".getBytes()), new String(s3)); + s4 = from2.get("key4".getBytes()); + assertEquals(new String("value4".getBytes()), new String(s4)); + */ + + // after merge from2 get data is success， traverse 0 times + byte[] s1 = from2.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s1)); + byte[] s2 = from2.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s2)); + + // this can not get key3 and key4 + assertNull(from.get("key3".getBytes())); + assertNull(from.get("key4".getBytes())); + } + + /** + * from: key1=>value1, key2=>value2, key3=>value31 + * from2: key3=>value32,key4=>value4 + * after merge: from2.mergeAhead(from); + * from2: key1=>value1, key2=>value2, key3=>value32, key4=>value4 + */ + @Test + public void testMergeOverride() { + tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( + "testSnapshotRoot-testMerge"); + revokingDatabase = context.getBean(SnapshotManager.class); + revokingDatabase.enable(); + revokingDatabase.add(tronDatabase.getRevokingDB()); + + // linklist is: from2 -> from -> root + SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); + SnapshotImpl from = getSnapshotImplIns(root); + from.put("key1".getBytes(), "value1".getBytes()); + from.put("key2".getBytes(), "value2".getBytes()); + from.put("key3".getBytes(), "value31".getBytes()); + + SnapshotImpl from2 = getSnapshotImplIns(from); + from2.put("key3".getBytes(), "value32".getBytes()); + from2.put("key4".getBytes(), "value4".getBytes()); + // do mergeAhead + from2.mergeAhead(from); + + // after merge from2 get data is success， traverse 0 times + byte[] s1 = from2.get("key1".getBytes()); + assertEquals(new String("value1".getBytes()), new String(s1)); + byte[] s2 = from2.get("key2".getBytes()); + assertEquals(new String("value2".getBytes()), new String(s2)); + byte[] s3 = from2.get("key3".getBytes()); + assertEquals(new String("value32".getBytes()), new String(s3)); + byte[] s4 = from2.get("key4".getBytes()); + assertEquals(new String("value4".getBytes()), new String(s4)); + } + + /** + * The constructor of SnapshotImpl is not public + * so reflection is used to construct the object here. + */ + private SnapshotImpl getSnapshotImplIns(Snapshot snapshot) { + Class clazz = SnapshotImpl.class; + try { + Constructor constructor = clazz.getDeclaredConstructor(Snapshot.class); + constructor.setAccessible(true); + return (SnapshotImpl) constructor.newInstance(snapshot); + } catch (Exception e) { + e.printStackTrace(); + } + return null; + } + +} From 9430e133df6b7618183a8689b5f5144c272225dd Mon Sep 17 00:00:00 2001 From: chaozhu Date: Fri, 1 Sep 2023 09:32:28 +0800 Subject: [PATCH 7/7] feature(mechanism): add unit test for properties db optimization --- .../org/tron/core/db2/SnapshotImplTest.java | 28 +++++++------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java index c40aaf760e0..aab6f656b1f 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java @@ -32,6 +32,12 @@ public void init() { Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); + + tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( + "testSnapshotRoot-testMerge"); + revokingDatabase = context.getBean(SnapshotManager.class); + revokingDatabase.enable(); + revokingDatabase.add(tronDatabase.getRevokingDB()); } @After @@ -39,6 +45,9 @@ public void removeDb() { Args.clearParam(); context.destroy(); FileUtil.deleteDir(new File("output_revokingStore_test")); + + tronDatabase.close(); + revokingDatabase.shutdown(); } /** @@ -51,15 +60,9 @@ public void removeDb() { */ @Test public void testMergeRoot() { - tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( - "testSnapshotRoot-testMerge"); - revokingDatabase = context.getBean(SnapshotManager.class); - revokingDatabase.enable(); - revokingDatabase.add(tronDatabase.getRevokingDB()); - // linklist is: from -> root SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); - root.setOptimized(true); + //root.setOptimized(true); root.put("key1".getBytes(), "value1".getBytes()); root.put("key2".getBytes(), "value2".getBytes()); @@ -87,11 +90,6 @@ public void testMergeRoot() { */ @Test public void testMergeAhead() { - tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( - "testSnapshotRoot-testMerge"); - revokingDatabase = context.getBean(SnapshotManager.class); - revokingDatabase.enable(); - revokingDatabase.add(tronDatabase.getRevokingDB()); // linklist is: from2 -> from -> root SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); @@ -160,12 +158,6 @@ public void testMergeAhead() { */ @Test public void testMergeOverride() { - tronDatabase = new RevokingDbWithCacheNewValueTest.TestRevokingTronStore( - "testSnapshotRoot-testMerge"); - revokingDatabase = context.getBean(SnapshotManager.class); - revokingDatabase.enable(); - revokingDatabase.add(tronDatabase.getRevokingDB()); - // linklist is: from2 -> from -> root SnapshotRoot root = new SnapshotRoot(tronDatabase.getDb()); SnapshotImpl from = getSnapshotImplIns(root);