From 3aa86e3424c09844850d1704a3ad4a553d089475 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Wed, 13 Jan 2021 18:30:40 +0800 Subject: [PATCH 01/20] =?UTF-8?q?=E8=A1=A8=E5=88=9B=E5=BB=BA=E3=80=81?= =?UTF-8?q?=E8=A1=A8=E5=A4=B4=E8=A7=A3=E6=9E=90=E4=BB=A5=E5=8F=8A=E9=A1=B5?= =?UTF-8?q?=E5=A4=B4=E8=A7=A3=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 10 +- .../kaaass/rumbase/dataitem/ItemManager.java | 14 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 242 ++++++++++++++++++ ...{FileException.java => ItemException.java} | 6 +- .../dataitem/mock/MockItemStorage.java | 11 +- .../rumbase/dataitem/IItemStorageTest.java | 55 ++-- 6 files changed, 299 insertions(+), 39 deletions(-) create mode 100644 src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java rename src/main/java/net/kaaass/rumbase/dataitem/exception/{FileException.java => ItemException.java} (74%) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 2f57144..52cb62c 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -1,7 +1,9 @@ package net.kaaass.rumbase.dataitem; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.List; /** @@ -16,7 +18,7 @@ public interface IItemStorage { * @param item 数据项 * @return 返回数据项的UUID */ - long insertItem(byte[] item); + long insertItem(TransactionContext txContext, byte[] item) throws IOException; /** * 插入一个有UUID的数据项，唯一使用的地方是日志恢复时使用 @@ -28,7 +30,7 @@ public interface IItemStorage { * @param item 数据项 * @param uuid 编号 */ - void insertItemWithUuid(byte[] item, long uuid); + void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid); /** * 通过UUID查询数据项 @@ -55,7 +57,7 @@ public interface IItemStorage { * @param item 数据项 * @throws UUIDException 没有找到对应UUID的异常 */ - void updateItemByUuid(long uuid, byte[] item) throws UUIDException; + void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException; /** * 获得数据项存储的元数据（可以用于头） @@ -69,7 +71,7 @@ public interface IItemStorage { * * @param metadata 头信息 */ - void setMetadata(byte[] metadata); + void setMetadata(TransactionContext txContext,byte[] metadata); /** * 清理多余的数据项，空间清理时使用。 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index ff4064d..9e6b33c 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -1,8 +1,13 @@ package net.kaaass.rumbase.dataitem; -import net.kaaass.rumbase.dataitem.exception.FileException; + import net.kaaass.rumbase.dataitem.mock.MockItemStorage; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.transaction.TransactionContext; +import net.kaaass.rumbase.transaction.mock.MockTransactionContext; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -22,7 +27,7 @@ public class ItemManager { * @param fileName 文件名 * @return 数据项管理器，用于管理数据项 */ - public static IItemStorage fromFile(String fileName) throws FileException { + public static IItemStorage fromFile(String fileName) throws FileException, IOException, PageException { String errorFileName = "error.db"; if (errorFileName.equals(fileName)) { throw new FileException(2); @@ -42,16 +47,17 @@ public static IItemStorage fromFile(String fileName) throws FileException { * * @param fileName 文件名 * @param metadata 上层提供的表头信息 + * @param txContext 对应的事务名 * @return 数据项管理器 * @throws FileException 想新建的文件已经存在的异常 */ - public static IItemStorage createFile(String fileName, byte[] metadata) throws FileException { + public static IItemStorage createFile(TransactionContext txContext ,String fileName, byte[] metadata) throws FileException, IOException, PageException { // 如果文件已经存在，那么就抛出文件已存在异常 if (maps.containsKey(fileName)) { throw new FileException(1); } else { // 若文件不存在，则创建文件。 - IItemStorage iItemStorage = MockItemStorage.ofNewFile(fileName, metadata); + IItemStorage iItemStorage = MockItemStorage.ofNewFile(txContext,fileName, metadata); maps.put(fileName, iItemStorage); return iItemStorage; } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java new file mode 100644 index 0000000..d4c5160 --- /dev/null +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -0,0 +1,242 @@ +package net.kaaass.rumbase.dataitem; + +import com.igormaznitsa.jbbp.JBBPParser; +import com.igormaznitsa.jbbp.mapper.Bin; +import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.Page; +import net.kaaass.rumbase.page.PageManager; +import net.kaaass.rumbase.page.PageStorage; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.IRecoveryStorage; +import net.kaaass.rumbase.transaction.TransactionContext; + +import java.io.IOException; +import java.util.List; + +/** + * 数据项管理器的具体实现 + * + *

+ * 每个表的头信息都是一致的，为 + * |头信息标志位：1234(共4字节)|当前可用的第一个空闲页(2字节)|是否写入表头信息(1字节)，写入为123|头信息所对应的UUID(8字节) + * + * 同时 + *

+ * @author kaito + */ +public class ItemStorage implements IItemStorage { + + private String fileName; + /** + * 当前第一个空闲的页，用于插入时作为起始页来进行操作。 + */ + private int tempFreePage; + /** + * 表信息头对应的UUID + */ + private long headerUuid; + /** + * 内部维护一个对应该文件的页管理器 + */ + private PageStorage pageStorage; + /** + * 维护一个日志管理器 + */ + private IRecoveryStorage recoveryStorage; + + + public ItemStorage(String fileName, int tempFreePage, long headerUuid,PageStorage pageStorage) { + this.fileName = fileName; + this.tempFreePage = tempFreePage; + this.headerUuid = headerUuid; + this.pageStorage = pageStorage; + } + + /** + * 判断有没有头部标志 + * + *

+ * 获取表头的前四个字节数据，若是1234则表示是表头，解析后面的数据，否则就认为该表没有被初始化 + *

+ * @param header 第一页的Page对象 + * @return 是否是表的第一页 + */ + private static boolean checkTableHeader(Page header){ + var data = header.getData(); + byte[] flag = new byte[4]; + try { + data.read(flag); + } catch (IOException e) { + e.printStackTrace(); + } + return flag[0] == 1 && flag[1] == 2 && flag[2] == 3 && flag[3] == 4; + } + + /** + * 打开相应的文件并读取头信息，如果没有对应的头信息就进行初始化 + * @param fileName 文件名 + * @return 解析或新建得到的数据项管理器对象 + */ + public static IItemStorage ofFile(String fileName) throws FileException, IOException, PageException { + var pageStorage = PageManager.fromFile(fileName); + var header = pageStorage.get(0); + if (checkTableHeader(header)){ + // 如果表头标志存在，就解析对应表头信息 + var h = JBBPParser.prepare("long headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;").parse(header.getData()).mapTo(new TableHeader()); + return new ItemStorage(fileName,h.tempFreePage,h.headerUuid,pageStorage); + }else { + // 若表头标志不存在，就初始化对应的表信息。 + // 只初始化headerFlag和tempFreePage，表头信息位置统一由setMetadata来实现 + byte[] bytes = new byte[]{1,2,3,4,0,1}; + header.patchData(0,bytes); + return new ItemStorage(fileName,1,0,pageStorage); + } + } + + /** + * 创建新的表，初始化相应数据，并且保存头信息。 + * @param fileName 文件名 + * @param metadata 表头信息 + * @return 数据项管理器 + */ + public static IItemStorage ofNewFile(TransactionContext txContext ,String fileName,byte[] metadata) throws IOException, FileException, PageException { + var pageStorage = ItemStorage.ofFile(fileName); + pageStorage.setMetadata(txContext,metadata); + return pageStorage; + } + + private PageHeader getPageHeader(Page page) throws IOException { + var data = page.getData(); + var pageHeader = JBBPParser.prepare("long pageFlag;int pageId;long lsn;int leftSpace;int recordNumber;" + + "Item[recordNumber]{int size;long uuid;int offset;}").parse(data).mapTo(new PageHeader()); + return pageHeader; + } + + + @Override + public long insertItem(TransactionContext txContext, byte[] item) throws IOException { + var page = pageStorage.get(this.tempFreePage); + var pageHeader = getPageHeader(page); + + return 0; + } + + @Override + public void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid) { + + } + + @Override + public byte[] queryItemByUuid(long uuid) throws UUIDException { + return new byte[0]; + } + + @Override + public List listItemByPageId(int pageId) { + return null; + } + + @Override + public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException { + + } + + @Override + public byte[] getMetadata() { + return new byte[0]; + } + + @Override + public void setMetadata(TransactionContext txContext,byte[] metadata) { + + } + + @Override + public void removeItems(List uuids) { + + } +} + +/** + * 表头 + */ +class TableHeader { + /** + * 是否是表头的标志 + */ + @Bin + long headerFlag; + /** + * 第一个可使用的空闲页编号 + */ + @Bin + int tempFreePage; + /** + * 是否有表头信息 + */ + @Bin + byte hasHeaderInfo; + /** + * 表头信息对应的UUID + */ + @Bin + long headerUuid; +} + +/** + * 每个数据项对应的相关信息 + */ +class Item{ + /** + * 数据项大小 + */ + @Bin + int size; + /** + * 数据项编号 + */ + @Bin + long uuid; + /** + * 页内偏移 + */ + @Bin + int offset; +} + +/** + * 页头 + */ +class PageHeader{ + /** + * 页头标志 + */ + @Bin + long pageFlag; + /** + * 当前页号 + */ + @Bin + int pageId; + /** + * 日志记录编号 + */ + @Bin + long lsn; + /** + * 剩余空间大小 + */ + @Bin + int leftSpace; + /** + * 页内记录总数 + */ + @Bin + int recordNumber; + /** + * 页内记录的相关信息 + */ + @Bin + Item[] item; +} \ No newline at end of file diff --git a/src/main/java/net/kaaass/rumbase/dataitem/exception/FileException.java b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java similarity index 74% rename from src/main/java/net/kaaass/rumbase/dataitem/exception/FileException.java rename to src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java index 429a233..b87b578 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/exception/FileException.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java @@ -6,17 +6,17 @@ import java.util.Map; /** - * 在创建、打开数据库时 + * 对数据库进行解析时出现的异常错误 * * @author kaito */ -public class FileException extends RumbaseException { +public class ItemException extends RumbaseException { public static final Map REASONS = new HashMap<>() {{ put(1, "要创建的文件已存在"); put(2, "查找的文件不存在"); }}; - public FileException(int subID) { + public ItemException(int subID) { super(6001, subID, REASONS.get(subID)); } } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index a4f555a..1a6e86f 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -3,6 +3,7 @@ import lombok.Data; import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.transaction.TransactionContext; import java.util.*; @@ -72,14 +73,14 @@ public static IItemStorage ofFile(String fileName) { * @param tableHeader 表头信息 * @return 返回数据项管理器 */ - public static IItemStorage ofNewFile(String fileName, byte[] tableHeader) { + public static IItemStorage ofNewFile(TransactionContext txContext ,String fileName, byte[] tableHeader) { // TODO: 因为是新建的文件，所以需要给文件头写入头信息数据。 return new MockItemStorage(fileName, 0, 0); } @Override - public long insertItem(byte[] item) { + public long insertItem(TransactionContext txContext,byte[] item) { Random ran = new Random(); long r = ran.nextLong(); maps.put(r, item); @@ -87,7 +88,7 @@ public long insertItem(byte[] item) { } @Override - public void insertItemWithUuid(byte[] item, long uuid) { + public void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid) { maps.put(uuid, item); } @@ -106,7 +107,7 @@ public List listItemByPageId(int pageId) { } @Override - public void updateItemByUuid(long uuid, byte[] item) throws UUIDException { + public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException { if (maps.containsKey(uuid)) { maps.put(uuid, item); } else { @@ -120,7 +121,7 @@ public byte[] getMetadata() { } @Override - public void setMetadata(byte[] metadata) { + public void setMetadata(TransactionContext txContext, byte[] metadata) { this.meta = metadata; } diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index a78b7fc..b1a3774 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -2,9 +2,13 @@ import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; -import net.kaaass.rumbase.dataitem.exception.FileException; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.transaction.TransactionContext; +import net.kaaass.rumbase.transaction.mock.MockTransactionContext; +import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.List; @@ -22,10 +26,9 @@ public class IItemStorageTest extends TestCase { /** * 测试能否从已有文件中解析得到数据项管理器 */ - public void testGetFromFile() throws FileException { + public void testGetFromFile() throws FileException, IOException, PageException { String fileName = "testGetFromFile.db"; var itemStorage = ItemManager.fromFile(fileName); - // 如果表中没有对应的文件，那么就抛出错误 String failFileName = "error.db"; try { @@ -39,18 +42,19 @@ public void testGetFromFile() throws FileException { * 测试能否新建文件并得到数据项管理器 */ public void testCreateFile() { + TransactionContext txContext = new MockTransactionContext(); String fileName = "testCreateFile.db"; byte[] metadata = new byte[1024]; // 第一次执行的时候，表中没有数据，不会报错 try { - var iItemStorage = ItemManager.createFile(fileName, metadata); + var iItemStorage = ItemManager.createFile(txContext,fileName, metadata); } catch (Exception e) { e.printStackTrace(); } try { - IItemStorage iItemStorage = ItemManager.createFile(fileName, metadata); - } catch (FileException e) { + IItemStorage iItemStorage = ItemManager.createFile(txContext,fileName, metadata); + } catch (Exception e) { log.error("Exception Error :", e); } } @@ -58,11 +62,12 @@ public void testCreateFile() { /** * 进行插入的测试 */ - public void testInsert() throws FileException { + public void testInsert() throws FileException, IOException, PageException { String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - long uuid = iItemStorage.insertItem(bytes); + TransactionContext txContext = new MockTransactionContext(); + long uuid = iItemStorage.insertItem(txContext,bytes); try { assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); } catch (UUIDException e) { @@ -73,13 +78,14 @@ public void testInsert() throws FileException { /** * 对插入一个已分配UUID的测试 */ - public void testInsertWithUUID() throws FileException { + public void testInsertWithUUID() throws FileException, IOException, PageException { String fileName = "testInsertWithUUID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; long uuid = 50; + TransactionContext txContext = new MockTransactionContext(); // 第一次插入，表中没有该UUID，可以正常执行 - iItemStorage.insertItemWithUuid(bytes, uuid); + iItemStorage.insertItemWithUuid(txContext,bytes, uuid); try { assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); } catch (UUIDException e) { @@ -87,18 +93,19 @@ public void testInsertWithUUID() throws FileException { } // 第二次插入 - iItemStorage.insertItemWithUuid(bytes, uuid); + iItemStorage.insertItemWithUuid(txContext,bytes, uuid); } /** * 对查询进行测试 */ - public void testQuery() throws FileException { + public void testQuery() throws FileException, IOException, PageException { String fileName = "testQuery.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - long uuid = iItemStorage.insertItem(bytes); + TransactionContext txContext = new MockTransactionContext(); + long uuid = iItemStorage.insertItem(txContext,bytes); // 查询可以正常执行 try { assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); @@ -120,14 +127,15 @@ public void testQuery() throws FileException { /** * 获取整个页的数据项进行测试 */ - public void testQueryByPageID() throws FileException { + public void testQueryByPageID() throws FileException, IOException, PageException { String fileName = "testQueryByPageID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - long uuid = iItemStorage.insertItem(bytes); + TransactionContext txContext = new MockTransactionContext(); + long uuid = iItemStorage.insertItem(txContext,bytes); byte[] bytes1 = new byte[]{2, 3, 4, 5}; - long uuid1 = iItemStorage.insertItem(bytes1); + long uuid1 = iItemStorage.insertItem(txContext,bytes1); Comparator comparator = new Comparator() { @Override @@ -162,15 +170,16 @@ public int compare(byte[] o1, byte[] o2) { /** * 对更新进行测试 */ - public void testUpdate() throws FileException { + public void testUpdate() throws FileException, IOException, PageException { String fileName = "testUpdate.db"; + TransactionContext txContext = new MockTransactionContext(); IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - long uuid = iItemStorage.insertItem(bytes); + long uuid = iItemStorage.insertItem(txContext,bytes); // 正常情况下进行修改 byte[] result = new byte[]{2, 3, 4, 5}; try { - iItemStorage.updateItemByUuid(uuid, result); + iItemStorage.updateItemByUuid(txContext,uuid, result); byte[] bs = iItemStorage.queryItemByUuid(uuid); assertEquals(bs, result); } catch (UUIDException e) { @@ -182,7 +191,7 @@ public void testUpdate() throws FileException { s += 1; } try { - iItemStorage.updateItemByUuid(s, result); + iItemStorage.updateItemByUuid(txContext,s, result); } catch (UUIDException e) { e.printStackTrace(); } @@ -191,12 +200,12 @@ public void testUpdate() throws FileException { /** * 测试修改和获取表头信息 */ - public void testMeta() throws FileException { + public void testMeta() throws FileException, IOException, PageException { String fileName = "testMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] result = new byte[]{1, 2, 3, 4}; - - iItemStorage.setMetadata(result); + TransactionContext txContext = new MockTransactionContext(); + iItemStorage.setMetadata(txContext,result); byte[] bs = iItemStorage.getMetadata(); assertEquals(result, bs); From 88ca20ca6555bb409a9c05e30893c053266696b9 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Wed, 13 Jan 2021 22:13:16 +0800 Subject: [PATCH 02/20] =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E5=92=8C=E6=8F=92=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 3 +- .../kaaass/rumbase/dataitem/ItemManager.java | 4 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 83 +++++++++++++++---- .../rumbase/dataitem/IItemStorageTest.java | 10 +-- 4 files changed, 77 insertions(+), 23 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 52cb62c..49143c9 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -1,6 +1,7 @@ package net.kaaass.rumbase.dataitem; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.transaction.TransactionContext; import java.io.IOException; @@ -18,7 +19,7 @@ public interface IItemStorage { * @param item 数据项 * @return 返回数据项的UUID */ - long insertItem(TransactionContext txContext, byte[] item) throws IOException; + long insertItem(TransactionContext txContext, byte[] item) throws IOException, PageException; /** * 插入一个有UUID的数据项，唯一使用的地方是日志恢复时使用 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index 9e6b33c..bf2ae89 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -36,7 +36,7 @@ public static IItemStorage fromFile(String fileName) throws FileException, IOExc if (maps.containsKey(fileName)) { return maps.get(fileName); } else { - IItemStorage iItemStorage = MockItemStorage.ofFile(fileName); + IItemStorage iItemStorage = ItemStorage.ofFile(fileName); maps.put(fileName, iItemStorage); return iItemStorage; } @@ -57,7 +57,7 @@ public static IItemStorage createFile(TransactionContext txContext ,String fileN throw new FileException(1); } else { // 若文件不存在，则创建文件。 - IItemStorage iItemStorage = MockItemStorage.ofNewFile(txContext,fileName, metadata); + IItemStorage iItemStorage = ItemStorage.ofNewFile(txContext,fileName, metadata); maps.put(fileName, iItemStorage); return iItemStorage; } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index d4c5160..3fc76e2 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -1,6 +1,7 @@ package net.kaaass.rumbase.dataitem; import com.igormaznitsa.jbbp.JBBPParser; +import com.igormaznitsa.jbbp.io.JBBPOut; import com.igormaznitsa.jbbp.mapper.Bin; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.Page; @@ -11,15 +12,17 @@ import net.kaaass.rumbase.recovery.IRecoveryStorage; import net.kaaass.rumbase.transaction.TransactionContext; +import javax.swing.text.html.Option; import java.io.IOException; import java.util.List; +import java.util.Optional; /** * 数据项管理器的具体实现 * *

* 每个表的头信息都是一致的，为 - * |头信息标志位：1234(共4字节)|当前可用的第一个空闲页(2字节)|是否写入表头信息(1字节)，写入为123|头信息所对应的UUID(8字节) + * |头信息标志位：1234(共4字节)|当前可用的第一个空闲页(4字节)|是否写入表头信息(1字节)，写入为123|头信息所对应的UUID(8字节) * * 同时 *

@@ -83,12 +86,12 @@ public static IItemStorage ofFile(String fileName) throws FileException, IOExcep var header = pageStorage.get(0); if (checkTableHeader(header)){ // 如果表头标志存在，就解析对应表头信息 - var h = JBBPParser.prepare("long headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;").parse(header.getData()).mapTo(new TableHeader()); + var h = JBBPParser.prepare("int headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;").parse(header.getData()).mapTo(new TableHeader()); return new ItemStorage(fileName,h.tempFreePage,h.headerUuid,pageStorage); }else { // 若表头标志不存在，就初始化对应的表信息。 // 只初始化headerFlag和tempFreePage，表头信息位置统一由setMetadata来实现 - byte[] bytes = new byte[]{1,2,3,4,0,1}; + byte[] bytes = new byte[]{1,2,3,4,0,0,0,1}; header.patchData(0,bytes); return new ItemStorage(fileName,1,0,pageStorage); } @@ -106,19 +109,55 @@ public static IItemStorage ofNewFile(TransactionContext txContext ,String fileNa return pageStorage; } - private PageHeader getPageHeader(Page page) throws IOException { - var data = page.getData(); - var pageHeader = JBBPParser.prepare("long pageFlag;int pageId;long lsn;int leftSpace;int recordNumber;" + - "Item[recordNumber]{int size;long uuid;int offset;}").parse(data).mapTo(new PageHeader()); - return pageHeader; + /** + * 检查一个页头标志位是否存在，表示其是否已经被初始化。若初始化则标志位为2345 + * @param page 页 + * @return 该页是否已经被初始化 + * @throws IOException + */ + private boolean checkPageHeader(Page page) throws IOException { + byte[] pageFlag = new byte[4]; + var n = page.getData().read(pageFlag); + return pageFlag[0] == 2 && pageFlag[1] == 3 && pageFlag[2] == 4 && pageFlag[3] == 5; } + private Optional getPageHeader(Page page) throws IOException { + if (!checkPageHeader(page)){ + // 如果页头信息正确，就解析得到页头信息的对象。 + var data = page.getData(); + byte[] s = new byte[40]; + var pageHeader = JBBPParser.prepare("int pageFlag;int pageId;long lsn;int leftSpace;int recordNumber;" + + "item [_]{int size;long uuid;int offset;}").parse(s); + var p = pageHeader.mapTo(new PageHeader()); + return Optional.of(p); + }else { + // 否则返回空，交由上层处理。 + return Optional.empty(); + } + } + + private PageHeader initPage(Page page,int pageId) throws IOException, PageException { + final byte[] bytes = JBBPOut.BeginBin(). + Byte(2,3,4,5). // 页头标志位 + Int(pageId). // pageId + Long(0). // 日志记录位置，以后若有日志记录点则使用 + Int(4072). // 剩余空间大小 + Int(0). // 记录数目 + End().toByteArray(); + page.patchData(0,bytes); + return new PageHeader(0,pageId,0,4072,0); + } @Override - public long insertItem(TransactionContext txContext, byte[] item) throws IOException { + public long insertItem(TransactionContext txContext, byte[] item) throws IOException, PageException { var page = pageStorage.get(this.tempFreePage); var pageHeader = getPageHeader(page); + if (pageHeader.isEmpty()){ + // 如果获取的页没有页头信息，则进行初始化。 + var p = initPage(page,this.tempFreePage); + }else { + } return 0; } @@ -166,7 +205,7 @@ class TableHeader { * 是否是表头的标志 */ @Bin - long headerFlag; + int headerFlag; /** * 第一个可使用的空闲页编号 */ @@ -192,17 +231,17 @@ class Item{ * 数据项大小 */ @Bin - int size; + public int size; /** * 数据项编号 */ @Bin - long uuid; + public long uuid; /** * 页内偏移 */ @Bin - int offset; + public int offset; } /** @@ -213,12 +252,12 @@ class PageHeader{ * 页头标志 */ @Bin - long pageFlag; + public int pageFlag; /** * 当前页号 */ @Bin - int pageId; + public int pageId; /** * 日志记录编号 */ @@ -239,4 +278,18 @@ class PageHeader{ */ @Bin Item[] item; + + public PageHeader() { } + + public PageHeader(int pageFlag, int pageId, long lsn, int leftSpace, int recordNumber) { + this.pageFlag = pageFlag; + this.pageId = pageId; + this.lsn = lsn; + this.leftSpace = leftSpace; + this.recordNumber = recordNumber; + } + + public Object newInstance(Class klazz){ + return klazz == Item.class ? new Item() : null; + } } \ No newline at end of file diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index b1a3774..b3c6110 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -68,11 +68,11 @@ public void testInsert() throws FileException, IOException, PageException { byte[] bytes = new byte[]{1, 2, 3, 4}; TransactionContext txContext = new MockTransactionContext(); long uuid = iItemStorage.insertItem(txContext,bytes); - try { - assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); - } catch (UUIDException e) { - e.printStackTrace(); - } +// try { +// assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); +// } catch (UUIDException e) { +// e.printStackTrace(); +// } } /** From f2827da14c109c705b6c37efc3f5c2d4f006a5a0 Mon Sep 17 00:00:00 2001 From: KAAAsS Date: Wed, 13 Jan 2021 22:24:46 +0800 Subject: [PATCH 03/20] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E7=BC=96=E8=AF=91?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/ItemStorage.java | 178 +++++++++--------- 1 file changed, 93 insertions(+), 85 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index 3fc76e2..f005c65 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -195,101 +195,109 @@ public void setMetadata(TransactionContext txContext,byte[] metadata) { public void removeItems(List uuids) { } -} -/** - * 表头 - */ -class TableHeader { - /** - * 是否是表头的标志 - */ - @Bin - int headerFlag; /** - * 第一个可使用的空闲页编号 + * 表头 */ - @Bin - int tempFreePage; - /** - * 是否有表头信息 - */ - @Bin - byte hasHeaderInfo; - /** - * 表头信息对应的UUID - */ - @Bin - long headerUuid; -} + public static class TableHeader { + /** + * 是否是表头的标志 + */ + @Bin + int headerFlag; + /** + * 第一个可使用的空闲页编号 + */ + @Bin + int tempFreePage; + /** + * 是否有表头信息 + */ + @Bin + byte hasHeaderInfo; + /** + * 表头信息对应的UUID + */ + @Bin + long headerUuid; -/** - * 每个数据项对应的相关信息 - */ -class Item{ - /** - * 数据项大小 - */ - @Bin - public int size; - /** - * 数据项编号 - */ - @Bin - public long uuid; - /** - * 页内偏移 - */ - @Bin - public int offset; -} + public Object newInstance(Class klazz){ + return klazz == TableHeader.class ? new TableHeader() : null; + } + } -/** - * 页头 - */ -class PageHeader{ - /** - * 页头标志 - */ - @Bin - public int pageFlag; /** - * 当前页号 + * 每个数据项对应的相关信息 */ - @Bin - public int pageId; - /** - * 日志记录编号 - */ - @Bin - long lsn; - /** - * 剩余空间大小 - */ - @Bin - int leftSpace; - /** - * 页内记录总数 - */ - @Bin - int recordNumber; + public static class Item{ + /** + * 数据项大小 + */ + @Bin + public int size; + /** + * 数据项编号 + */ + @Bin + public long uuid; + /** + * 页内偏移 + */ + @Bin + public int offset; + + public Object newInstance(Class klazz){ + return klazz == Item.class ? new Item() : null; + } + } + /** - * 页内记录的相关信息 + * 页头 */ - @Bin - Item[] item; + public static class PageHeader{ + /** + * 页头标志 + */ + @Bin + public int pageFlag; + /** + * 当前页号 + */ + @Bin + public int pageId; + /** + * 日志记录编号 + */ + @Bin + long lsn; + /** + * 剩余空间大小 + */ + @Bin + int leftSpace; + /** + * 页内记录总数 + */ + @Bin + int recordNumber; + /** + * 页内记录的相关信息 + */ + @Bin + Item[] item; - public PageHeader() { } + public PageHeader() { } - public PageHeader(int pageFlag, int pageId, long lsn, int leftSpace, int recordNumber) { - this.pageFlag = pageFlag; - this.pageId = pageId; - this.lsn = lsn; - this.leftSpace = leftSpace; - this.recordNumber = recordNumber; - } + public PageHeader(int pageFlag, int pageId, long lsn, int leftSpace, int recordNumber) { + this.pageFlag = pageFlag; + this.pageId = pageId; + this.lsn = lsn; + this.leftSpace = leftSpace; + this.recordNumber = recordNumber; + } - public Object newInstance(Class klazz){ - return klazz == Item.class ? new Item() : null; + public Object newInstance(Class klazz){ + return klazz == PageHeader.class ? new PageHeader() : null; + } } -} \ No newline at end of file +} From bc9f6fc033823f982317b5d59fdc422f0122bf1d Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Thu, 14 Jan 2021 12:36:00 +0800 Subject: [PATCH 04/20] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=A1=B9=E7=9A=84?= =?UTF-8?q?=E6=8F=92=E5=85=A5=E5=92=8C=E6=9F=A5=E8=AF=A2=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 2 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 225 +++++++++++++++--- .../rumbase/dataitem/IItemStorageTest.java | 12 +- 3 files changed, 209 insertions(+), 30 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 49143c9..0047181 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -40,7 +40,7 @@ public interface IItemStorage { * @return 数据项 * @throws UUIDException UUID找不到的异常 */ - byte[] queryItemByUuid(long uuid) throws UUIDException; + byte[] queryItemByUuid(long uuid) throws UUIDException, IOException; /** diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index f005c65..d15d4a1 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -15,7 +15,9 @@ import javax.swing.text.html.Option; import java.io.IOException; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Random; /** * 数据项管理器的具体实现 @@ -91,7 +93,10 @@ public static IItemStorage ofFile(String fileName) throws FileException, IOExcep }else { // 若表头标志不存在，就初始化对应的表信息。 // 只初始化headerFlag和tempFreePage，表头信息位置统一由setMetadata来实现 - byte[] bytes = new byte[]{1,2,3,4,0,0,0,1}; + var bytes = JBBPOut.BeginBin(). + Byte(1,2,3,4). + Int(1). + End().toByteArray(); header.patchData(0,bytes); return new ItemStorage(fileName,1,0,pageStorage); } @@ -122,12 +127,11 @@ private boolean checkPageHeader(Page page) throws IOException { } private Optional getPageHeader(Page page) throws IOException { - if (!checkPageHeader(page)){ + if (checkPageHeader(page)){ // 如果页头信息正确，就解析得到页头信息的对象。 var data = page.getData(); - byte[] s = new byte[40]; - var pageHeader = JBBPParser.prepare("int pageFlag;int pageId;long lsn;int leftSpace;int recordNumber;" + - "item [_]{int size;long uuid;int offset;}").parse(s); + var pageHeader = JBBPParser.prepare("int pageFlag;long lsn;int leftSpace;int recordNumber;" + + "item [recordNumber]{int uuid;int offset;}").parse(data); var p = pageHeader.mapTo(new PageHeader()); return Optional.of(p); }else { @@ -136,29 +140,94 @@ private Optional getPageHeader(Page page) throws IOException { } } - private PageHeader initPage(Page page,int pageId) throws IOException, PageException { + private PageHeader initPage(Page page) throws IOException, PageException { final byte[] bytes = JBBPOut.BeginBin(). Byte(2,3,4,5). // 页头标志位 - Int(pageId). // pageId Long(0). // 日志记录位置，以后若有日志记录点则使用 Int(4072). // 剩余空间大小 Int(0). // 记录数目 End().toByteArray(); page.patchData(0,bytes); - return new PageHeader(0,pageId,0,4072,0); + return new PageHeader(0,0,4072,0); + } + + /** + * 修改当前第一个可用页 + */ + private void addTempFreePage() throws IOException, PageException { + this.tempFreePage += 1; + var page = pageStorage.get(0); + page.pin(); + var tempFreePage = JBBPOut.BeginBin(). + Int(this.tempFreePage) + .End().toByteArray(); + page.patchData(4,tempFreePage); + page.unpin(); } @Override public long insertItem(TransactionContext txContext, byte[] item) throws IOException, PageException { - var page = pageStorage.get(this.tempFreePage); - var pageHeader = getPageHeader(page); - if (pageHeader.isEmpty()){ + var page = getPage(this.tempFreePage); + var pageHeaderOp = getPageHeader(page); + if (pageHeaderOp.isEmpty()){ // 如果获取的页没有页头信息，则进行初始化。 - var p = initPage(page,this.tempFreePage); + pageHeaderOp = Optional.of(initPage(page)); + } + var pageHeader = pageHeaderOp.get(); + if (pageHeader.leftSpace - Math.min(item.length,MAX_RECORD_SIZE) <= MIN_LEFT_SPACE){ + // 如果剩余空间过小的话，就切换到下一个页进行，同时修改表头信息.并且，若数据过大则使用拉链，所以取512和数据大小较小的 + addTempFreePage(); + releasePage(page); + return insertItem(txContext,item); }else { + // 剩余空间足够，则插入 + int rnd = Math.abs(new Random().nextInt()); + long s= this.tempFreePage; + long uuid = ((s << 32) + (long)(rnd)); + // 保证uuid不重复 + while (checkUuidExist(uuid)){ + rnd = Math.abs(new Random().nextInt()); + uuid = ((s << 32) + (long)(rnd)); + } + insertToPage(page,pageHeader,txContext,item,rnd); + releasePage(page); + return uuid; + } + } + /** + * 将数据插入到页内对应位置，并修改页头信息 + */ + private void insertToPage(Page page,PageHeader pageHeader,TransactionContext txContext,byte[] item,int rnd) throws IOException, PageException { + if (item.length < MAX_RECORD_SIZE){ + int offset = 0 ; + if (pageHeader.recordNumber == 0){ + // 如果页没有元素的话 + offset = 4095 - item.length - DATA_EXTRA_SIZE; + }else { + // 如果页内有插入的数据，则读取其offset并推算自己的offset + offset = pageHeader.item[pageHeader.recordNumber-1].offset - item.length - DATA_EXTRA_SIZE; + } + + //修改数据项头信息 + var i = JBBPOut.BeginBin(). + Int(rnd). + Int(offset). + End().toByteArray(); + page.patchData(ITEM_OFFSET + pageHeader.recordNumber * ITEM_SIZE ,i); + //修改数据信息 + var data = JBBPOut.BeginBin(). + Byte(123). + Int(item.length). + Byte(item).End().toByteArray(); + page.patchData(offset,data); + //修改页头信息 + var headerInfo = JBBPOut.BeginBin(). + Int(pageHeader.leftSpace - item.length - DATA_EXTRA_SIZE). + Int(pageHeader.recordNumber + 1). + End().toByteArray(); + page.patchData(LEFT_SPACE_OFFSET,headerInfo); } - return 0; } @Override @@ -166,9 +235,77 @@ public void insertItemWithUuid(TransactionContext txContext,byte[] item, long uu } + /** + * 根据uuid获取page + */ + private Page getPage(long uuid){ + var pageId = uuid >> 32; + Page page = pageStorage.get(pageId); + page.pin(); + return page; + } + + private Page getPage(int pageId){ + Page page = pageStorage.get(pageId); + page.pin(); + return page; + } + + /** + * 释放page的操作 + * @param page + */ + private void releasePage(Page page){ + page.unpin(); + } + /** + * 检查uuid是否存在 + * @param uuid + * @return + */ + private boolean checkUuidExist(long uuid) throws IOException { + var page = getPage(uuid); + int id = (int) (uuid & 0xFFFFFFFF); + var pageHeader = getPageHeader(page); + if (pageHeader.isEmpty()){ + return false; + }else { + for (var item : pageHeader.get().item){ + if (id == item.uuid){ + return true; + } + } + } + return false; + } + @Override - public byte[] queryItemByUuid(long uuid) throws UUIDException { - return new byte[0]; + public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { + if (checkUuidExist(uuid)){ + var page = getPage(uuid); + var header = getPageHeader(page); + if (header.isEmpty()){ + releasePage(page); + throw new UUIDException(2); + }else { + // 遍历所有的item读取数据 + var items = header.get().item; + int id = (int) (uuid & 0xFFFFFFFF); + for (var item : items){ + if (item.uuid == id){ + int offset = item.offset; + var data = page.getData(); + data.skip(offset); + var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); + releasePage(page); + return content.data; + } + } + } + return new byte[0]; + }else { + throw new UUIDException(2); + } } @Override @@ -196,6 +333,36 @@ public void removeItems(List uuids) { } + /** + * 数据项记录的大小 + */ + final static int ITEM_SIZE = 8; + + /** + * 数据项记录的起始偏移 + */ + final static int ITEM_OFFSET = 20; + /** + * 记录数据项标志和数据项大小额外占用的空间 + */ + final static int DATA_EXTRA_SIZE = 5; + /** + * 页内空闲空间的页内偏移 + */ + final static int LEFT_SPACE_OFFSET = 12; + /** + * 页的大小 + */ + final static int PAGE_SIZE = 4096; + /** + * 单个数据项的最大值，超过的话使用拉链 + */ + final static int MAX_RECORD_SIZE = 512; + /** + * 每个页保留的大小 + */ + final static int MIN_LEFT_SPACE = 409; + /** * 表头 */ @@ -230,16 +397,11 @@ public Object newInstance(Class klazz){ * 每个数据项对应的相关信息 */ public static class Item{ - /** - * 数据项大小 - */ - @Bin - public int size; /** * 数据项编号 */ @Bin - public long uuid; + public int uuid; /** * 页内偏移 */ @@ -260,11 +422,7 @@ public static class PageHeader{ */ @Bin public int pageFlag; - /** - * 当前页号 - */ - @Bin - public int pageId; + /** * 日志记录编号 */ @@ -288,9 +446,8 @@ public static class PageHeader{ public PageHeader() { } - public PageHeader(int pageFlag, int pageId, long lsn, int leftSpace, int recordNumber) { + public PageHeader(int pageFlag,long lsn, int leftSpace, int recordNumber) { this.pageFlag = pageFlag; - this.pageId = pageId; this.lsn = lsn; this.leftSpace = leftSpace; this.recordNumber = recordNumber; @@ -300,4 +457,16 @@ public Object newInstance(Class klazz){ return klazz == PageHeader.class ? new PageHeader() : null; } } + + public static class ItemContent{ + @Bin + byte type; + @Bin + int size; + @Bin + byte[] data; + public Object newInstance(Class klazz){ + return klazz == ItemContent.class ? new ItemContent() : null; + } + } } diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index b3c6110..aea2bfa 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.List; @@ -68,6 +69,10 @@ public void testInsert() throws FileException, IOException, PageException { byte[] bytes = new byte[]{1, 2, 3, 4}; TransactionContext txContext = new MockTransactionContext(); long uuid = iItemStorage.insertItem(txContext,bytes); + + long uuid2 = iItemStorage.insertItem(txContext,bytes); + + long uuid3 = iItemStorage.insertItem(txContext,bytes); // try { // assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); // } catch (UUIDException e) { @@ -106,9 +111,14 @@ public void testQuery() throws FileException, IOException, PageException { byte[] bytes = new byte[]{1, 2, 3, 4}; TransactionContext txContext = new MockTransactionContext(); long uuid = iItemStorage.insertItem(txContext,bytes); + long uuid2 = iItemStorage.insertItem(txContext,bytes); + long uuid3 = iItemStorage.insertItem(txContext,bytes); // 查询可以正常执行 try { - assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); + var item = iItemStorage.queryItemByUuid(uuid); + assertTrue(Arrays.equals(bytes,item)); + var item3 = iItemStorage.queryItemByUuid(uuid3); + assertTrue(Arrays.equals(bytes,item3)); } catch (UUIDException e) { e.printStackTrace(); } From 6f0d1539ed2d7af991086d0bd9be71d850716e18 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Thu, 14 Jan 2021 12:53:28 +0800 Subject: [PATCH 05/20] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E6=95=B4=E9=A1=B5?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 2 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 45 +++++++++++++------ .../rumbase/dataitem/IItemStorageTest.java | 6 ++- 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 0047181..699d0f0 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -49,7 +49,7 @@ public interface IItemStorage { * @param pageId 页号 * @return list的一堆数据项 */ - List listItemByPageId(int pageId); + List listItemByPageId(int pageId) throws IOException; /** * 根据UUID更新数据项 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index d15d4a1..f7b33d4 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -14,10 +14,8 @@ import javax.swing.text.html.Option; import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Random; +import java.io.InputStream; +import java.util.*; /** * 数据项管理器的具体实现 @@ -198,7 +196,7 @@ public long insertItem(TransactionContext txContext, byte[] item) throws IOExcep /** * 将数据插入到页内对应位置，并修改页头信息 */ - private void insertToPage(Page page,PageHeader pageHeader,TransactionContext txContext,byte[] item,int rnd) throws IOException, PageException { + private synchronized void insertToPage(Page page,PageHeader pageHeader,TransactionContext txContext,byte[] item,int rnd) throws IOException, PageException { if (item.length < MAX_RECORD_SIZE){ int offset = 0 ; if (pageHeader.recordNumber == 0){ @@ -293,12 +291,7 @@ public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { int id = (int) (uuid & 0xFFFFFFFF); for (var item : items){ if (item.uuid == id){ - int offset = item.offset; - var data = page.getData(); - data.skip(offset); - var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); - releasePage(page); - return content.data; + return parseData(page,item); } } } @@ -308,9 +301,35 @@ public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { } } + /** + * 通过偏移量解析得到数据 + * @param page 页 + * @param item 一个数据项记录 + * @return 解析得到的数据 + */ + private byte[] parseData(Page page,Item item) throws IOException { + int offset = item.offset; + var data = page.getData(); + data.skip(offset); + var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); + releasePage(page); + return content.data; + } + @Override - public List listItemByPageId(int pageId) { - return null; + public List listItemByPageId(int pageId) throws IOException { + var page = getPage(pageId); + List bytes = new ArrayList<>(); + var pageHeaderOp = getPageHeader(page); + if (pageHeaderOp.isPresent()){ + var pageHeader = pageHeaderOp.get(); + for (var item : pageHeader.item){ + var data = parseData(page,item); + bytes.add(data); + } + } + releasePage(page); + return bytes; } @Override diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index aea2bfa..3f99e4c 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -169,9 +169,11 @@ public int compare(byte[] o1, byte[] o2) { bs.add(bytes1); bs.sort(comparator); // 获取pageID对应的数据项，在这里Mock是获取所有list中的数据 - var result = iItemStorage.listItemByPageId(0); + var result = iItemStorage.listItemByPageId(1); result.sort(comparator); - assertEquals(bs, result); + for (int i= 0 ; i < bs.size() ; i++){ + assertTrue(Arrays.equals(bs.get(i),result.get(i))); + } } catch (Exception e) { e.printStackTrace(); } From 9d7af659cfc579bf7902aec2c22b6b8bd25b7ef1 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Thu, 14 Jan 2021 18:55:34 +0800 Subject: [PATCH 06/20] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=A4=B4=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E6=B7=BB=E5=8A=A0=20=E4=BB=A5=E5=8F=8A=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 9 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 128 ++++++++++++++---- .../dataitem/exception/ItemException.java | 3 +- 3 files changed, 106 insertions(+), 34 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 699d0f0..a598177 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -1,5 +1,6 @@ package net.kaaass.rumbase.dataitem; +import net.kaaass.rumbase.dataitem.exception.ItemException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.transaction.TransactionContext; @@ -31,7 +32,7 @@ public interface IItemStorage { * @param item 数据项 * @param uuid 编号 */ - void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid); + void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid) throws IOException, PageException; /** * 通过UUID查询数据项 @@ -58,21 +59,21 @@ public interface IItemStorage { * @param item 数据项 * @throws UUIDException 没有找到对应UUID的异常 */ - void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException; + void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException, IOException, PageException; /** * 获得数据项存储的元数据（可以用于头） * * @return 元数据 */ - byte[] getMetadata(); + byte[] getMetadata() throws IOException, ItemException, UUIDException; /** * 设置数据项存储的元数据（可以用于头） * * @param metadata 头信息 */ - void setMetadata(TransactionContext txContext,byte[] metadata); + void setMetadata(TransactionContext txContext,byte[] metadata) throws IOException, PageException; /** * 清理多余的数据项，空间清理时使用。 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index f7b33d4..d85bb32 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -3,6 +3,7 @@ import com.igormaznitsa.jbbp.JBBPParser; import com.igormaznitsa.jbbp.io.JBBPOut; import com.igormaznitsa.jbbp.mapper.Bin; +import net.kaaass.rumbase.dataitem.exception.ItemException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.Page; import net.kaaass.rumbase.page.PageManager; @@ -84,9 +85,11 @@ private static boolean checkTableHeader(Page header){ public static IItemStorage ofFile(String fileName) throws FileException, IOException, PageException { var pageStorage = PageManager.fromFile(fileName); var header = pageStorage.get(0); + header.pin(); if (checkTableHeader(header)){ // 如果表头标志存在，就解析对应表头信息 - var h = JBBPParser.prepare("int headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;").parse(header.getData()).mapTo(new TableHeader()); + var h = parseHeader(header); + header.unpin(); return new ItemStorage(fileName,h.tempFreePage,h.headerUuid,pageStorage); }else { // 若表头标志不存在，就初始化对应的表信息。 @@ -96,6 +99,7 @@ public static IItemStorage ofFile(String fileName) throws FileException, IOExcep Int(1). End().toByteArray(); header.patchData(0,bytes); + header.unpin(); return new ItemStorage(fileName,1,0,pageStorage); } } @@ -112,6 +116,34 @@ public static IItemStorage ofNewFile(TransactionContext txContext ,String fileNa return pageStorage; } + private static int getRndByUuid(long uuid){ + return (int)(uuid & 0xFFFFFFFF); + } + + /** + * 解析表头数据 + * @return 解析得到的表头对象 + * + */ + private static TableHeader parseHeader(Page page) throws IOException { + return JBBPParser.prepare("int headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;"). + parse(page.getData()).mapTo(new TableHeader()); + } + + /** + * 通过偏移量解析得到数据 + * @param page 页 + * @param item 一个数据项记录 + * @return 解析得到的数据 + */ + private static byte[] parseData(Page page,Item item) throws IOException { + int offset = item.offset; + var data = page.getData(); + data.skip(offset); + var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); + return content.data; + } + /** * 检查一个页头标志位是否存在，表示其是否已经被初始化。若初始化则标志位为2345 * @param page 页 @@ -164,7 +196,7 @@ private void addTempFreePage() throws IOException, PageException { } @Override - public long insertItem(TransactionContext txContext, byte[] item) throws IOException, PageException { + public synchronized long insertItem(TransactionContext txContext, byte[] item) throws IOException, PageException { var page = getPage(this.tempFreePage); var pageHeaderOp = getPageHeader(page); if (pageHeaderOp.isEmpty()){ @@ -196,7 +228,7 @@ public long insertItem(TransactionContext txContext, byte[] item) throws IOExcep /** * 将数据插入到页内对应位置，并修改页头信息 */ - private synchronized void insertToPage(Page page,PageHeader pageHeader,TransactionContext txContext,byte[] item,int rnd) throws IOException, PageException { + private void insertToPage(Page page,PageHeader pageHeader,TransactionContext txContext,byte[] item,int rnd) throws IOException, PageException { if (item.length < MAX_RECORD_SIZE){ int offset = 0 ; if (pageHeader.recordNumber == 0){ @@ -229,8 +261,20 @@ private synchronized void insertToPage(Page page,PageHeader pageHeader,Transacti } @Override - public void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid) { - + public synchronized void insertItemWithUuid(TransactionContext txContext,byte[] item, long uuid) throws IOException, PageException { + if (!checkUuidExist(uuid)){ + // 若不存在，则要恢复 + var page = getPage(uuid); + var pageHeaderOp = getPageHeader(page); + if (pageHeaderOp.isEmpty()){ + // 如果获取的页没有页头信息，则进行初始化。 + pageHeaderOp = Optional.of(initPage(page)); + } + int rnd = getRndByUuid(uuid); + insertToPage(page,pageHeaderOp.get(),txContext,item,rnd); + releasePage(page); + } + // 若存在则不需要恢复，直接返回 } /** @@ -263,7 +307,7 @@ private void releasePage(Page page){ */ private boolean checkUuidExist(long uuid) throws IOException { var page = getPage(uuid); - int id = (int) (uuid & 0xFFFFFFFF); + int id = getRndByUuid(uuid); var pageHeader = getPageHeader(page); if (pageHeader.isEmpty()){ return false; @@ -288,10 +332,12 @@ public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { }else { // 遍历所有的item读取数据 var items = header.get().item; - int id = (int) (uuid & 0xFFFFFFFF); + int id = getRndByUuid(uuid); for (var item : items){ if (item.uuid == id){ - return parseData(page,item); + var s =parseData(page,item); + releasePage(page); + return s; } } } @@ -301,20 +347,7 @@ public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { } } - /** - * 通过偏移量解析得到数据 - * @param page 页 - * @param item 一个数据项记录 - * @return 解析得到的数据 - */ - private byte[] parseData(Page page,Item item) throws IOException { - int offset = item.offset; - var data = page.getData(); - data.skip(offset); - var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); - releasePage(page); - return content.data; - } + @Override public List listItemByPageId(int pageId) throws IOException { @@ -333,18 +366,48 @@ public List listItemByPageId(int pageId) throws IOException { } @Override - public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException { - + public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item) throws UUIDException, IOException, PageException { + var page = getPage(uuid); + if (checkUuidExist(uuid)){ + var pageHeader = getPageHeader(page); + var items = pageHeader.get().item; + for(var i : items){ + if (i.uuid == getRndByUuid(uuid)){ + var offset = i.offset; + page.patchData(offset,item); + releasePage(page); + return; + } + } + }else{ + releasePage(page); + throw new UUIDException(2); + } } @Override - public byte[] getMetadata() { - return new byte[0]; + public byte[] getMetadata() throws IOException, ItemException, UUIDException { + var page = getPage(0); + var header = parseHeader(page); + if (checkTableHeader(page) && header.hasHeaderInfo == hasHeader){ + // 若表头已经被初始化并且有标志位的话，就说明有表头信息，进行获取. + var h = queryItemByUuid(header.headerUuid); + releasePage(page); + return h; + }else { + throw new ItemException(1); + } } @Override - public void setMetadata(TransactionContext txContext,byte[] metadata) { - + public void setMetadata(TransactionContext txContext,byte[] metadata) throws IOException, PageException { + var page = getPage(0); + var headerUuid = insertItem(txContext,metadata); + var bytes = JBBPOut.BeginBin(). + Byte(hasHeader). + Long(headerUuid).End().toByteArray(); + page.patchData(HeaderOffset,bytes); + releasePage(page); } @Override @@ -352,6 +415,15 @@ public void removeItems(List uuids) { } + /** + * 头部标志的偏移 + */ + final static int HeaderOffset = 8; + + /** + * 表头判断是否有表头数据 + */ + final static byte hasHeader = 123; /** * 数据项记录的大小 */ diff --git a/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java index b87b578..98837d2 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java @@ -12,8 +12,7 @@ */ public class ItemException extends RumbaseException { public static final Map REASONS = new HashMap<>() {{ - put(1, "要创建的文件已存在"); - put(2, "查找的文件不存在"); + put(1, "没有相应表头信息"); }}; public ItemException(int subID) { From 50a72511c4109610579a8bff983840fe32de1214 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Thu, 14 Jan 2021 20:22:30 +0800 Subject: [PATCH 07/20] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=A1=B9=E5=A2=9E?= =?UTF-8?q?=E6=94=B9=E6=9F=A5=E4=BB=A5=E5=8F=8A=E5=A4=B4=E9=83=A8=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=AE=8C=E6=88=90=EF=BC=8C=E6=B7=BB=E5=8A=A0=E5=B9=B6?= =?UTF-8?q?=E5=8F=91=E6=8F=92=E5=85=A5=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 4 +- .../kaaass/rumbase/dataitem/ItemManager.java | 5 - .../kaaass/rumbase/dataitem/ItemStorage.java | 100 +++++++++------- .../dataitem/exception/ItemException.java | 1 + .../rumbase/dataitem/IItemStorageTest.java | 110 +++++++++++------- 5 files changed, 132 insertions(+), 88 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index a598177..1a20bc9 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -41,7 +41,7 @@ public interface IItemStorage { * @return 数据项 * @throws UUIDException UUID找不到的异常 */ - byte[] queryItemByUuid(long uuid) throws UUIDException, IOException; + byte[] queryItemByUuid(long uuid) throws UUIDException, IOException, ItemException; /** @@ -50,7 +50,7 @@ public interface IItemStorage { * @param pageId 页号 * @return list的一堆数据项 */ - List listItemByPageId(int pageId) throws IOException; + List listItemByPageId(int pageId) throws IOException, ItemException; /** * 根据UUID更新数据项 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index bf2ae89..2e09325 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -28,11 +28,6 @@ public class ItemManager { * @return 数据项管理器，用于管理数据项 */ public static IItemStorage fromFile(String fileName) throws FileException, IOException, PageException { - String errorFileName = "error.db"; - if (errorFileName.equals(fileName)) { - throw new FileException(2); - } - if (maps.containsKey(fileName)) { return maps.get(fileName); } else { diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index d85bb32..672d3c2 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -13,9 +13,7 @@ import net.kaaass.rumbase.recovery.IRecoveryStorage; import net.kaaass.rumbase.transaction.TransactionContext; -import javax.swing.text.html.Option; import java.io.IOException; -import java.io.InputStream; import java.util.*; /** @@ -23,9 +21,15 @@ * *

* 每个表的头信息都是一致的，为 - * |头信息标志位：1234(共4字节)|当前可用的第一个空闲页(4字节)|是否写入表头信息(1字节)，写入为123|头信息所对应的UUID(8字节) + * |头信息标志位：1 2 3 4(共4字节)|当前可用的第一个空闲页(4字节)|是否写入表头信息(1字节)，写入为123|头信息所对应的UUID(8字节) * - * 同时 + * 同时每个页都有相应的页头，页头格式为： + * |页头标志位 2 3 4 5(共4字节)|lsn来记录日志相关内容(8字节)|页剩余空间大小(4字节)|页内数据项个数n(4字节)|每个数据项标志(n*8字节)| + * + * 数据项标志为 |uuid后面的随机数(4字节)|在页内偏移offset(4字节)| + * + * 每个数据项的内容为|标志位，表示有没有拉链等特殊情况(1字节)|数据长度m(4字节)|数据内容(m字节)| + * TODO : |(若有拉链的话，则记录下一个uuid位置)8字节| *

* @author kaito */ @@ -116,10 +120,40 @@ public static IItemStorage ofNewFile(TransactionContext txContext ,String fileNa return pageStorage; } + /** + * 根据uuid获取后面的随机位置 + */ private static int getRndByUuid(long uuid){ return (int)(uuid & 0xFFFFFFFF); } + /** + * 根据uuid获取page + */ + private Page getPage(long uuid){ + var pageId = uuid >> 32; + Page page = pageStorage.get(pageId); + page.pin(); + return page; + } + + /** + * 根据pageId获取Page + */ + private Page getPage(int pageId){ + Page page = pageStorage.get(pageId); + page.pin(); + return page; + } + + /** + * 释放page的操作 + * @param page + */ + private void releasePage(Page page){ + page.unpin(); + } + /** * 解析表头数据 * @return 解析得到的表头对象 @@ -136,11 +170,14 @@ private static TableHeader parseHeader(Page page) throws IOException { * @param item 一个数据项记录 * @return 解析得到的数据 */ - private static byte[] parseData(Page page,Item item) throws IOException { + private static byte[] parseData(Page page,Item item) throws IOException, ItemException { int offset = item.offset; var data = page.getData(); data.skip(offset); var content = JBBPParser.prepare("byte type;int size;byte[size] data;").parse(data).mapTo(new ItemContent()); + if (content.type != NORMAL_DATA){ + throw new ItemException(2); + } return content.data; } @@ -238,7 +275,6 @@ private void insertToPage(Page page,PageHeader pageHeader,TransactionContext tx // 如果页内有插入的数据，则读取其offset并推算自己的offset offset = pageHeader.item[pageHeader.recordNumber-1].offset - item.length - DATA_EXTRA_SIZE; } - //修改数据项头信息 var i = JBBPOut.BeginBin(). Int(rnd). @@ -247,13 +283,13 @@ private void insertToPage(Page page,PageHeader pageHeader,TransactionContext tx page.patchData(ITEM_OFFSET + pageHeader.recordNumber * ITEM_SIZE ,i); //修改数据信息 var data = JBBPOut.BeginBin(). - Byte(123). + Byte(NORMAL_DATA). Int(item.length). Byte(item).End().toByteArray(); page.patchData(offset,data); //修改页头信息 var headerInfo = JBBPOut.BeginBin(). - Int(pageHeader.leftSpace - item.length - DATA_EXTRA_SIZE). + Int(pageHeader.leftSpace - item.length - DATA_EXTRA_SIZE - ITEM_SIZE). Int(pageHeader.recordNumber + 1). End().toByteArray(); page.patchData(LEFT_SPACE_OFFSET,headerInfo); @@ -277,29 +313,6 @@ public synchronized void insertItemWithUuid(TransactionContext txContext,byte[] // 若存在则不需要恢复，直接返回 } - /** - * 根据uuid获取page - */ - private Page getPage(long uuid){ - var pageId = uuid >> 32; - Page page = pageStorage.get(pageId); - page.pin(); - return page; - } - - private Page getPage(int pageId){ - Page page = pageStorage.get(pageId); - page.pin(); - return page; - } - - /** - * 释放page的操作 - * @param page - */ - private void releasePage(Page page){ - page.unpin(); - } /** * 检查uuid是否存在 * @param uuid @@ -322,7 +335,7 @@ private boolean checkUuidExist(long uuid) throws IOException { } @Override - public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { + public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException, ItemException { if (checkUuidExist(uuid)){ var page = getPage(uuid); var header = getPageHeader(page); @@ -350,7 +363,7 @@ public byte[] queryItemByUuid(long uuid) throws UUIDException, IOException { @Override - public List listItemByPageId(int pageId) throws IOException { + public List listItemByPageId(int pageId) throws IOException, ItemException { var page = getPage(pageId); List bytes = new ArrayList<>(); var pageHeaderOp = getPageHeader(page); @@ -374,7 +387,12 @@ public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item for(var i : items){ if (i.uuid == getRndByUuid(uuid)){ var offset = i.offset; - page.patchData(offset,item); + var bytes = JBBPOut.BeginBin(). + Byte(NORMAL_DATA). + Int(item.length) + .Byte(item) + .End().toByteArray(); + page.patchData(offset,bytes); releasePage(page); return; } @@ -389,7 +407,7 @@ public void updateItemByUuid(TransactionContext txContext,long uuid, byte[] item public byte[] getMetadata() throws IOException, ItemException, UUIDException { var page = getPage(0); var header = parseHeader(page); - if (checkTableHeader(page) && header.hasHeaderInfo == hasHeader){ + if (checkTableHeader(page) && header.hasHeaderInfo == HAS_HEADER){ // 若表头已经被初始化并且有标志位的话，就说明有表头信息，进行获取. var h = queryItemByUuid(header.headerUuid); releasePage(page); @@ -404,9 +422,9 @@ public void setMetadata(TransactionContext txContext,byte[] metadata) throws IOE var page = getPage(0); var headerUuid = insertItem(txContext,metadata); var bytes = JBBPOut.BeginBin(). - Byte(hasHeader). + Byte(HAS_HEADER). Long(headerUuid).End().toByteArray(); - page.patchData(HeaderOffset,bytes); + page.patchData(HEADER_OFFSET,bytes); releasePage(page); } @@ -415,15 +433,19 @@ public void removeItems(List uuids) { } + /** + * + */ + final static byte NORMAL_DATA = 121; /** * 头部标志的偏移 */ - final static int HeaderOffset = 8; + final static int HEADER_OFFSET = 8; /** * 表头判断是否有表头数据 */ - final static byte hasHeader = 123; + final static byte HAS_HEADER = 123; /** * 数据项记录的大小 */ diff --git a/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java index 98837d2..434fb60 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/exception/ItemException.java @@ -13,6 +13,7 @@ public class ItemException extends RumbaseException { public static final Map REASONS = new HashMap<>() {{ put(1, "没有相应表头信息"); + put(2,"所查找的数据项信息解析错误"); }}; public ItemException(int subID) { diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index 3f99e4c..88a99b1 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -2,6 +2,7 @@ import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; +import net.kaaass.rumbase.dataitem.exception.ItemException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; @@ -9,10 +10,7 @@ import net.kaaass.rumbase.transaction.mock.MockTransactionContext; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; +import java.util.*; /** * 对数据项管理部分进行测试 @@ -63,7 +61,7 @@ public void testCreateFile() { /** * 进行插入的测试 */ - public void testInsert() throws FileException, IOException, PageException { + public void testInsert() throws FileException, IOException, PageException, UUIDException, ItemException { String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -73,11 +71,9 @@ public void testInsert() throws FileException, IOException, PageException { long uuid2 = iItemStorage.insertItem(txContext,bytes); long uuid3 = iItemStorage.insertItem(txContext,bytes); -// try { -// assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); -// } catch (UUIDException e) { -// e.printStackTrace(); -// } + + assertTrue(Arrays.equals(bytes,iItemStorage.queryItemByUuid(uuid))); + } /** @@ -87,13 +83,15 @@ public void testInsertWithUUID() throws FileException, IOException, PageExceptio String fileName = "testInsertWithUUID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - long uuid = 50; + long s = 1; + int rnd = Math.abs(new Random().nextInt()); + long uuid = (s << 32) + rnd; TransactionContext txContext = new MockTransactionContext(); // 第一次插入，表中没有该UUID，可以正常执行 iItemStorage.insertItemWithUuid(txContext,bytes, uuid); try { - assertEquals(bytes, iItemStorage.queryItemByUuid(uuid)); - } catch (UUIDException e) { + assertTrue(Arrays.equals(bytes, iItemStorage.queryItemByUuid(uuid))); + } catch (UUIDException | ItemException e) { e.printStackTrace(); } @@ -103,34 +101,26 @@ public void testInsertWithUUID() throws FileException, IOException, PageExceptio } /** - * 对查询进行测试 + * 对插入大量数据进行测试 */ - public void testQuery() throws FileException, IOException, PageException { - String fileName = "testQuery.db"; + public void testManyInsert() throws FileException, IOException, PageException { + String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; TransactionContext txContext = new MockTransactionContext(); - long uuid = iItemStorage.insertItem(txContext,bytes); - long uuid2 = iItemStorage.insertItem(txContext,bytes); - long uuid3 = iItemStorage.insertItem(txContext,bytes); - // 查询可以正常执行 - try { - var item = iItemStorage.queryItemByUuid(uuid); - assertTrue(Arrays.equals(bytes,item)); - var item3 = iItemStorage.queryItemByUuid(uuid3); - assertTrue(Arrays.equals(bytes,item3)); - } catch (UUIDException e) { - e.printStackTrace(); - } - // 找一个跟UUID不同的，也就是说不在数据库中的UUID进行查询 - long s = 1; - if (s == uuid) { - s += 1; - } - try { - var b = iItemStorage.queryItemByUuid(s); - } catch (UUIDException e) { - log.error("Exception Error :", e); + for (int i = 0;i < 100 ; i ++){ + long uuid = iItemStorage.insertItem(txContext,bytes); + long uuid2 = iItemStorage.insertItem(txContext,bytes); + long uuid3 = iItemStorage.insertItem(txContext,bytes); + // 查询可以正常执行 + try { + var item = iItemStorage.queryItemByUuid(uuid); + assertTrue(Arrays.equals(bytes,item)); + var item3 = iItemStorage.queryItemByUuid(uuid3); + assertTrue(Arrays.equals(bytes,item3)); + } catch (UUIDException | ItemException e) { + e.printStackTrace(); + } } } @@ -179,6 +169,43 @@ public int compare(byte[] o1, byte[] o2) { } } + + class Insert implements Runnable{ + IItemStorage iItemStorage; + TransactionContext txContext; + + public Insert(IItemStorage iItemStorage, TransactionContext txContext) { + this.iItemStorage = iItemStorage; + this.txContext = txContext; + } + @Override + public void run() { + var bytes = new byte[]{1,2,3,4}; + try { + long uuid = iItemStorage.insertItem(txContext,bytes); + assertTrue(Arrays.equals(bytes,iItemStorage.queryItemByUuid(uuid))); + } catch (Exception e){ + e.printStackTrace(); + } + } + } + + /** + * 测试并发下插入是否有问题 + */ + public void testSynInsert() throws IOException, FileException, PageException { + String fileName = "testInsert.db"; + IItemStorage iItemStorage = ItemManager.fromFile(fileName); + byte[] bytes = new byte[]{1, 2, 3, 4}; + TransactionContext txContext = new MockTransactionContext(); + new Thread(new Insert(iItemStorage,txContext)).start(); + new Thread(new Insert(iItemStorage,txContext)).start(); + new Thread(new Insert(iItemStorage,txContext)).start(); + new Thread(new Insert(iItemStorage,txContext)).start(); + + } + + /** * 对更新进行测试 */ @@ -193,8 +220,8 @@ public void testUpdate() throws FileException, IOException, PageException { try { iItemStorage.updateItemByUuid(txContext,uuid, result); byte[] bs = iItemStorage.queryItemByUuid(uuid); - assertEquals(bs, result); - } catch (UUIDException e) { + assertTrue(Arrays.equals(bs, result)); + } catch (UUIDException | ItemException e) { e.printStackTrace(); } // 修改一个不存在的UUID @@ -212,15 +239,14 @@ public void testUpdate() throws FileException, IOException, PageException { /** * 测试修改和获取表头信息 */ - public void testMeta() throws FileException, IOException, PageException { + public void testMeta() throws FileException, IOException, PageException, UUIDException, ItemException { String fileName = "testMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] result = new byte[]{1, 2, 3, 4}; TransactionContext txContext = new MockTransactionContext(); iItemStorage.setMetadata(txContext,result); byte[] bs = iItemStorage.getMetadata(); - assertEquals(result, bs); - + assertTrue(Arrays.equals(result,bs)); } } From 0c54154d0d2e1f40bf33df89068ecb623845974e Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Thu, 14 Jan 2021 23:32:07 +0800 Subject: [PATCH 08/20] =?UTF-8?q?=E6=97=A5=E5=BF=97=E7=AE=A1=E7=90=86?= =?UTF-8?q?=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/ItemStorage.java | 2 +- .../rumbase/recovery/RecoveryManager.java | 19 +++- .../rumbase/recovery/RecoveryStorage.java | 86 +++++++++++++++++++ .../recovery/exception/LogException.java | 21 +++++ .../rumbase/recovery/IRecoveryTest.java | 17 +--- 5 files changed, 125 insertions(+), 20 deletions(-) create mode 100644 src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java create mode 100644 src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index 672d3c2..36d3441 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -434,7 +434,7 @@ public void removeItems(List uuids) { } /** - * + * 判断对应offset的位置是不是一个完整的数据 */ final static byte NORMAL_DATA = 121; /** diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index 4fbc18a..a846723 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -1,7 +1,12 @@ package net.kaaass.rumbase.recovery; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.recovery.mock.MockRecoveryStorage; +import java.io.IOException; + /** * 日志恢复的管理器，用来对每个数据库文件进行恢复 * @@ -9,13 +14,21 @@ */ public class RecoveryManager { /** - * 对某个数据库文件进行恢复，并且返回对应的日志管理器 + * 对某个数据库文件进行恢复 * * @param fileName 文件名 * @return 数据库日志管理器 */ - public static IRecoveryStorage recovery(String fileName) { + public static void recovery(String fileName) { // TODO:对数据进行恢复 - return new MockRecoveryStorage(); + + } + + public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException { + return RecoveryStorage.ofFile(fileName + ".log"); + } + + public static IRecoveryStorage createRecoveryStorage(String fileName) throws IOException, FileException, PageException { + return RecoveryStorage.ofNewFile(fileName + ".log"); } } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java new file mode 100644 index 0000000..44e7d93 --- /dev/null +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -0,0 +1,86 @@ +package net.kaaass.rumbase.recovery; + +import com.igormaznitsa.jbbp.JBBPParser; +import com.igormaznitsa.jbbp.io.JBBPOut; +import com.igormaznitsa.jbbp.mapper.Bin; +import net.kaaass.rumbase.dataitem.ItemManager; +import net.kaaass.rumbase.page.PageManager; +import net.kaaass.rumbase.page.PageStorage; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; + +import java.io.IOException; +import java.util.List; + +/** + * 日志记录相关内容 + * + * @author kaito + */ +public class RecoveryStorage implements IRecoveryStorage { + + /** + * 读取日志文件并且解析得到日志管理器 + * @param fileName + * @return 日志管理器 + */ + public static IRecoveryStorage ofFile(String fileName) throws FileException, IOException, LogException { + return new RecoveryStorage(); + } + + /** + * 创建日志文件 + * @param fileName 文件名 + * @return + */ + public static IRecoveryStorage ofNewFile(String fileName) throws FileException, IOException, PageException { + return new RecoveryStorage(); + } + + @Override + public void begin(int xid, List snapshots) { + + } + + @Override + public void rollback(int xid) { + + } + + @Override + public void commit(int xid) { + + } + + @Override + public void insert(int xid, long uuid, byte[] item) { + + } + + @Override + public void update(int xid, long uuid, byte[] item) { + + } + + @Override + public void updateMeta(int xid, long metaUUID) { + + } + + @Override + public List getContent() { + return null; + } + + /** + * 日志头信息 + * TODO: 记录点等特殊要求 + */ + public static class LogHeader{ + @Bin + int header; + } + final private static int HEADER = 1234; + +} diff --git a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java new file mode 100644 index 0000000..d15b0a5 --- /dev/null +++ b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java @@ -0,0 +1,21 @@ +package net.kaaass.rumbase.recovery.exception; + +import net.kaaass.rumbase.exception.RumbaseException; + +import java.util.HashMap; +import java.util.Map; + +public class LogException extends RumbaseException { + /** + * 构造Rumbase异常 + * + */ + + public static final Map REASONS = new HashMap() {{ + put(1, "日志文件无法解析"); + }}; + + public LogException(int subId) { + super(9001,subId,REASONS.get(subId)); + } +} diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index a94f553..118b2d2 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -9,27 +9,12 @@ import static org.junit.Assert.assertArrayEquals; /** - * TODO 文档 + * 对日志进行保存和恢复 */ @Slf4j public class IRecoveryTest extends TestCase { public void testBegin() { - IRecoveryStorage iRecoveryStorage = RecoveryManager.recovery("user.db"); - List l = new ArrayList<>(); - int xid = 1; - l.add(3); - l.add(2); - iRecoveryStorage.begin(xid, l); - var content = iRecoveryStorage.getContent(); - String beginStr = "begin " + xid; - String snapStr = "snap " + l.toString(); - List result = new ArrayList<>(); - result.add(beginStr.getBytes()); - result.add(snapStr.getBytes()); - assertEquals(result.size(), content.size()); - assertArrayEquals(result.get(0), content.get(0)); - assertArrayEquals(result.get(1), content.get(1)); } } From 4f45f1862aeda9413651e9e6ce15e8909cbb7508 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Fri, 15 Jan 2021 16:46:50 +0800 Subject: [PATCH 09/20] =?UTF-8?q?=E6=97=A5=E5=BF=97=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 22 +++- .../kaaass/rumbase/dataitem/ItemManager.java | 27 ++++- .../kaaass/rumbase/dataitem/ItemStorage.java | 86 ++++++++++++-- .../dataitem/mock/MockItemStorage.java | 18 ++- .../rumbase/recovery/IRecoveryStorage.java | 16 +-- .../rumbase/recovery/RecoveryManager.java | 4 +- .../rumbase/recovery/RecoveryStorage.java | 106 +++++++++++++++--- .../recovery/mock/MockRecoveryStorage.java | 2 +- .../rumbase/dataitem/IItemStorageTest.java | 23 ++-- 9 files changed, 256 insertions(+), 48 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 70c6b17..3c90ec2 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -2,6 +2,7 @@ import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.transaction.TransactionContext; import java.util.List; @@ -12,14 +13,27 @@ * @author kaito */ public interface IItemStorage { + + /** + * 将uuid对应的页强制写回 + */ + public void flush(long uuid) throws FileException; /** * 插入数据项 * + * @param txContext 事务上下文 * @param item 数据项 * @return 返回数据项的UUID */ long insertItem(TransactionContext txContext, byte[] item); + /** + * 不用日志进行插入，用于日志的管理 + * @param item 数据项 + * @return uuid + */ + long insertItemWithoutLog(byte[] item); + /** * 插入一个有UUID的数据项，唯一使用的地方是日志恢复时使用 *

@@ -30,7 +44,7 @@ public interface IItemStorage { * @param item 数据项 * @param uuid 编号 */ - void insertItemWithUuid(TransactionContext txContext, byte[] item, long uuid); + void insertItemWithUuid(byte[] item, long uuid); /** * 通过UUID查询数据项 @@ -73,6 +87,12 @@ public interface IItemStorage { */ void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException; + /** + * 不使用日志设置元数据 + * @param metadata 头信息 + */ + void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException; + /** * 清理多余的数据项，空间清理时使用。 * diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index 609fff6..0ce485a 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -3,6 +3,7 @@ import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import java.io.IOException; @@ -25,7 +26,7 @@ public class ItemManager { * @param fileName 文件名 * @return 数据项管理器，用于管理数据项 */ - public static IItemStorage fromFile(String fileName) throws FileException, IOException, PageException { + public static IItemStorage fromFile(String fileName) throws FileException, IOException, PageException, LogException { if (maps.containsKey(fileName)) { return maps.get(fileName); } else { @@ -35,6 +36,17 @@ public static IItemStorage fromFile(String fileName) throws FileException, IOExc } } + public static IItemStorage fromFileWithoutLog(String fileName) throws FileException, IOException, PageException, LogException { + if (maps.containsKey(fileName)) { + return maps.get(fileName); + } else { + IItemStorage iItemStorage = ItemStorage.ofFileWithoutLog(fileName); + maps.put(fileName, iItemStorage); + return iItemStorage; + } + } + + /** * 新建一个数据库，并且将上层提供的头信息写入。 * @@ -44,7 +56,7 @@ public static IItemStorage fromFile(String fileName) throws FileException, IOExc * @return 数据项管理器 * @throws FileException 想新建的文件已经存在的异常 */ - public static IItemStorage createFile(TransactionContext txContext, String fileName, byte[] metadata) throws FileException, IOException, PageException { + public static IItemStorage createFile(TransactionContext txContext, String fileName, byte[] metadata) throws FileException, IOException, PageException, LogException { // 如果文件已经存在，那么就抛出文件已存在异常 if (maps.containsKey(fileName)) { throw new FileException(1); @@ -54,6 +66,17 @@ public static IItemStorage createFile(TransactionContext txContext, String fileN maps.put(fileName, iItemStorage); return iItemStorage; } + } + public static IItemStorage createFileWithoutLog(String fileName, byte[] metadata) throws FileException, IOException, PageException, LogException { + // 如果文件已经存在，那么就抛出文件已存在异常 + if (maps.containsKey(fileName)) { + throw new FileException(1); + } else { + // 若文件不存在，则创建文件。 + IItemStorage iItemStorage = ItemStorage.ofNewFileWithoutLog(fileName, metadata); + maps.put(fileName, iItemStorage); + return iItemStorage; + } } } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index 7cb3217..e2e023e 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -12,6 +12,8 @@ import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.IRecoveryStorage; +import net.kaaass.rumbase.recovery.RecoveryManager; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import java.io.IOException; @@ -19,6 +21,7 @@ import java.util.List; import java.util.Optional; import java.util.Random; +import java.util.logging.LogManager; /** * 数据项管理器的具体实现 @@ -60,6 +63,14 @@ public class ItemStorage implements IItemStorage { private IRecoveryStorage recoveryStorage; + public ItemStorage(String fileName, int tempFreePage, long headerUuid, PageStorage pageStorage,IRecoveryStorage iRecoveryStorage) { + this.fileName = fileName; + this.tempFreePage = tempFreePage; + this.headerUuid = headerUuid; + this.pageStorage = pageStorage; + this.recoveryStorage = iRecoveryStorage; + } + public ItemStorage(String fileName, int tempFreePage, long headerUuid, PageStorage pageStorage) { this.fileName = fileName; this.tempFreePage = tempFreePage; @@ -94,7 +105,40 @@ private static boolean checkTableHeader(Page header) { * @param fileName 文件名 * @return 解析或新建得到的数据项管理器对象 */ - public static IItemStorage ofFile(String fileName) throws FileException, PageException { + public static IItemStorage ofFile(String fileName) throws FileException, PageException, IOException, LogException { + var pageStorage = PageManager.fromFile(fileName); + var header = pageStorage.get(0); + header.pin(); + if (checkTableHeader(header)) { + // 如果表头标志存在，就解析对应表头信息 + var h = parseHeader(header); + header.unpin(); + var logStorage = RecoveryManager.getRecoveryStorage(fileName); + return new ItemStorage(fileName, h.tempFreePage, h.headerUuid, pageStorage,logStorage); + } else { + // 若表头标志不存在，就初始化对应的表信息。 + // 只初始化headerFlag和tempFreePage，表头信息位置统一由setMetadata来实现 + byte[] bytes; + try { + bytes = JBBPOut.BeginBin(). + Byte(1, 2, 3, 4). + Int(1). + End().toByteArray(); + } catch (IOException e) { + header.unpin(); + throw new PageCorruptedException(1, e); + } + header.patchData(0, bytes); + header.unpin(); + var logStorage = RecoveryManager.createRecoveryStorage(fileName); + return new ItemStorage(fileName, 1, 0, pageStorage,logStorage); + } + } + + /** + * 不使用日志打开文件 + */ + public static IItemStorage ofFileWithoutLog(String fileName) throws FileException, PageException, IOException, LogException { var pageStorage = PageManager.fromFile(fileName); var header = pageStorage.get(0); header.pin(); @@ -129,12 +173,21 @@ public static IItemStorage ofFile(String fileName) throws FileException, PageExc * @param metadata 表头信息 * @return 数据项管理器 */ - public static IItemStorage ofNewFile(TransactionContext txContext, String fileName, byte[] metadata) throws IOException, FileException, PageException { + public static IItemStorage ofNewFile(TransactionContext txContext, String fileName, byte[] metadata) throws IOException, FileException, PageException, LogException { var pageStorage = ItemStorage.ofFile(fileName); pageStorage.setMetadata(txContext, metadata); return pageStorage; } + /** + * 不使用日志的创建文件 + */ + public static IItemStorage ofNewFileWithoutLog(String fileName, byte[] metadata) throws IOException, LogException, FileException, PageException { + var pageStorage = ItemStorage.ofFileWithoutLog(fileName); + pageStorage.setMetadataWithoutLog(metadata); + return pageStorage; + } + /** * 根据uuid获取后面的随机位置 */ @@ -275,8 +328,19 @@ private void addTempFreePage() { } } + @Override + public void flush(long uuid) throws FileException { + var page = getPage(uuid); + page.flush(); + } + @Override public synchronized long insertItem(TransactionContext txContext, byte[] item) { + long uuid = insertItemWithoutLog(item); + return uuid; + } + @Override + public synchronized long insertItemWithoutLog(byte[] item) { var page = getPage(this.tempFreePage); try { var pageHeaderOp = getPageHeader(page); @@ -288,7 +352,7 @@ public synchronized long insertItem(TransactionContext txContext, byte[] item) { if (pageHeader.leftSpace - Math.min(item.length, MAX_RECORD_SIZE) <= MIN_LEFT_SPACE) { // 如果剩余空间过小的话，就切换到下一个页进行，同时修改表头信息.并且，若数据过大则使用拉链，所以取512和数据大小较小的 addTempFreePage(); - return insertItem(txContext, item); + return insertItemWithoutLog( item); } else { // 剩余空间足够，则插入 int rnd = Math.abs(new Random().nextInt()); @@ -299,7 +363,7 @@ public synchronized long insertItem(TransactionContext txContext, byte[] item) { rnd = Math.abs(new Random().nextInt()); uuid = ((s << 32) + (long) (rnd)); } - insertToPage(page, pageHeader, txContext, item, rnd); + insertToPage(page, pageHeader,item, rnd); return uuid; } } catch (PageCorruptedException e) { @@ -314,7 +378,7 @@ public synchronized long insertItem(TransactionContext txContext, byte[] item) { /** * 将数据插入到页内对应位置，并修改页头信息 */ - private void insertToPage(Page page, PageHeader pageHeader, TransactionContext txContext, byte[] item, int rnd) { + private void insertToPage(Page page, PageHeader pageHeader,byte[] item, int rnd) { if (item.length < MAX_RECORD_SIZE) { int offset = 0; if (pageHeader.recordNumber == 0) { @@ -354,7 +418,7 @@ private void insertToPage(Page page, PageHeader pageHeader, TransactionContext t } @Override - public synchronized void insertItemWithUuid(TransactionContext txContext, byte[] item, long uuid) { + public synchronized void insertItemWithUuid(byte[] item, long uuid) { if (!checkUuidExist(uuid)) { // 若不存在，则要恢复 var page = getPage(uuid); @@ -365,7 +429,7 @@ public synchronized void insertItemWithUuid(TransactionContext txContext, byte[] pageHeaderOp = Optional.of(initPage(page)); } int rnd = getRndByUuid(uuid); - insertToPage(page, pageHeaderOp.get(), txContext, item, rnd); + insertToPage(page, pageHeaderOp.get(), item, rnd); } catch (Exception e) { throw new PageCorruptedException(3); } finally { @@ -515,9 +579,15 @@ public byte[] getMetadata() { @Override public void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException { + setMetadataWithoutLog(metadata); + + } + + @Override + public void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { var page = getPage(0); try { - var headerUuid = insertItem(txContext, metadata); + var headerUuid = insertItemWithoutLog(metadata); var bytes = JBBPOut.BeginBin(). Byte(HAS_HEADER). Long(headerUuid).End().toByteArray(); diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index 19e6fd5..3ffe357 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -2,6 +2,7 @@ import lombok.Data; import net.kaaass.rumbase.dataitem.IItemStorage; +import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.transaction.TransactionContext; @@ -79,6 +80,11 @@ public static IItemStorage ofNewFile(TransactionContext txContext, String fileNa } + @Override + public void flush(long uuid) { + + } + @Override public long insertItem(TransactionContext txContext, byte[] item) { Random ran = new Random(); @@ -88,7 +94,12 @@ public long insertItem(TransactionContext txContext, byte[] item) { } @Override - public void insertItemWithUuid(TransactionContext txContext, byte[] item, long uuid) { + public long insertItemWithoutLog( byte[] item) { + return 0; + } + + @Override + public void insertItemWithUuid(byte[] item, long uuid) { maps.put(uuid, item); } @@ -125,6 +136,11 @@ public void setMetadata(TransactionContext txContext, byte[] metadata) { this.meta = metadata; } + @Override + public void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { + + } + @Override public void removeItems(List uuids) { diff --git a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java index 0684071..372e8a3 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java @@ -1,5 +1,8 @@ package net.kaaass.rumbase.recovery; +import net.kaaass.rumbase.page.exception.FileException; + +import java.io.IOException; import java.util.List; /** @@ -14,21 +17,21 @@ public interface IRecoveryStorage { * @param xid 事务编号 * @param snapshots 快照集合 */ - void begin(int xid, List snapshots); + void begin(int xid, List snapshots) throws IOException, FileException; /** * 记录事务失败回滚 * * @param xid */ - void rollback(int xid); + void rollback(int xid) throws IOException, FileException; /** * 记录事务完成 * * @param xid */ - void commit(int xid); + void commit(int xid) throws IOException, FileException; /** * 插入数据项的日志记录 @@ -37,16 +40,15 @@ public interface IRecoveryStorage { * @param uuid 数据项的对应编号 * @param item 插入的数据内容 */ - void insert(int xid, long uuid, byte[] item); + void insert(int xid, long uuid, byte[] item) throws IOException, FileException; /** * 更新数据项的日志记录 * * @param xid * @param uuid - * @param item */ - void update(int xid, long uuid, byte[] item); + void update(int xid, long uuid, byte[] item_before,byte[] item_after) throws IOException, FileException; /** * 更新数据项的日志头 @@ -54,7 +56,7 @@ public interface IRecoveryStorage { * @param xid * @param metaUUID 头信息的UUID */ - void updateMeta(int xid, long metaUUID); + void updateMeta(int xid, long metaUUID) throws IOException, FileException; /** * 模拟打印日志资料 diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index a846723..3957bca 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -24,11 +24,11 @@ public static void recovery(String fileName) { } - public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException { + public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException, PageException { return RecoveryStorage.ofFile(fileName + ".log"); } - public static IRecoveryStorage createRecoveryStorage(String fileName) throws IOException, FileException, PageException { + public static IRecoveryStorage createRecoveryStorage(String fileName) throws IOException, FileException, PageException, LogException { return RecoveryStorage.ofNewFile(fileName + ".log"); } } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index 44e7d93..7ca44cd 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -3,12 +3,15 @@ import com.igormaznitsa.jbbp.JBBPParser; import com.igormaznitsa.jbbp.io.JBBPOut; import com.igormaznitsa.jbbp.mapper.Bin; +import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.ItemManager; import net.kaaass.rumbase.page.PageManager; import net.kaaass.rumbase.page.PageStorage; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; +import net.kaaass.rumbase.transaction.TransactionContext; +import net.kaaass.rumbase.transaction.mock.MockTransactionContext; import java.io.IOException; import java.util.List; @@ -20,13 +23,20 @@ */ public class RecoveryStorage implements IRecoveryStorage { + private IItemStorage itemStorage; + + public RecoveryStorage(IItemStorage itemStorage) { + this.itemStorage = itemStorage; + } + /** * 读取日志文件并且解析得到日志管理器 * @param fileName * @return 日志管理器 */ - public static IRecoveryStorage ofFile(String fileName) throws FileException, IOException, LogException { - return new RecoveryStorage(); + public static IRecoveryStorage ofFile(String fileName) throws FileException, IOException, LogException, PageException { + var itemStorage = ItemManager.fromFileWithoutLog(fileName + ".log"); + return new RecoveryStorage(itemStorage); } /** @@ -34,37 +44,84 @@ public static IRecoveryStorage ofFile(String fileName) throws FileException, IOE * @param fileName 文件名 * @return */ - public static IRecoveryStorage ofNewFile(String fileName) throws FileException, IOException, PageException { - return new RecoveryStorage(); + public static IRecoveryStorage ofNewFile(String fileName) throws FileException, IOException, PageException, LogException { + var metadata = JBBPOut.BeginBin().Int(HEADER).End().toByteArray(); + var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log",metadata); + return new RecoveryStorage(itemStorage); } @Override - public void begin(int xid, List snapshots) { - + public void begin(int xid, List snapshots) throws IOException, FileException { + var jbbp = JBBPOut.BeginBin(). + String(TX_BEGIN). + Int(xid). + Int(snapshots.size()); + for ( var i : snapshots){ + jbbp = jbbp.Int(i); + } + var bytes = jbbp.End().toByteArray(); + var uuid = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(uuid); } @Override - public void rollback(int xid) { - + public void rollback(int xid) throws IOException, FileException { + var bytes = JBBPOut.BeginBin(). + String(TX_ABORT). + Int(xid). + End().toByteArray(); + var uuid = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(uuid); } @Override - public void commit(int xid) { - + public void commit(int xid) throws IOException, FileException { + var bytes = JBBPOut.BeginBin(). + String(TX_COMMIT). + Int(xid). + End().toByteArray(); + var uuid = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(uuid); } @Override - public void insert(int xid, long uuid, byte[] item) { + public void insert(int xid, long uuid, byte[] item) throws IOException, FileException { + var bytes = JBBPOut.BeginBin(). + String(INSERT_FLAG). + Int(xid). + Long(uuid). + Int(item.length). + Byte(item). + End().toByteArray(); + var id = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(id); } @Override - public void update(int xid, long uuid, byte[] item) { - + public void update(int xid, long uuid, byte[] item_before,byte[] item_after) throws IOException, FileException { + var bytes = JBBPOut.BeginBin(). + String(UPDATE_FLAG). + Int(xid). + Long(uuid). + Int(item_before.length). + Byte(item_before). + Int(item_after.length). + Byte(item_after). + End().toByteArray(); + var id = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(id); } @Override - public void updateMeta(int xid, long metaUUID) { + public void updateMeta(int xid, long metaUUID) throws IOException, FileException { + var bytes =JBBPOut.BeginBin(). + String(METADATA_UPDATE_FLAG). + Int(xid). + Long(metaUUID). + End().toByteArray(); + var uuid = itemStorage.insertItemWithoutLog(bytes); + itemStorage.flush(uuid); } @@ -81,6 +138,25 @@ public static class LogHeader{ @Bin int header; } - final private static int HEADER = 1234; + + /** + * 日志标志 + */ + final private static int HEADER = 4567; + + final private static String INSERT_FLAG = "i"; + + final private static String TX_BEGIN = "b"; + + final private static String TX_ABORT = "a"; + + final private static String TX_COMMIT = "c"; + + final private static String UPDATE_FLAG = "u"; + + final private static String METADATA_UPDATE_FLAG = "m"; + + + } diff --git a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java index fa8c4e9..461cf2e 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java @@ -49,7 +49,7 @@ public void insert(int xid, long uuid, byte[] item) { } @Override - public void update(int xid, long uuid, byte[] item) { + public void update(int xid, long uuid, byte[] item,byte[] item_after) { String updateStr = "update " + xid + " " + uuid + " "; // 对控制语句、数据两部分进行合并得到最终日志记录 byte[] first = updateStr.getBytes(); diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index 588a8e7..ca0bb4c 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -6,6 +6,7 @@ import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import java.io.IOException; @@ -29,7 +30,7 @@ public class IItemStorageTest extends TestCase { /** * 测试能否从已有文件中解析得到数据项管理器 */ - public void testGetFromFile() throws FileException, IOException, PageException { + public void testGetFromFile() throws FileException, IOException, PageException, LogException { String fileName = "testGetFromFile.db"; var itemStorage = ItemManager.fromFile(fileName); // 如果表中没有对应的文件，那么就抛出错误 @@ -44,7 +45,7 @@ public void testGetFromFile() throws FileException, IOException, PageException { /** * 测试能否新建文件并得到数据项管理器 */ - public void testCreateFile() throws IOException, FileException, PageException { + public void testCreateFile() throws IOException, FileException, PageException, LogException { TransactionContext txContext = TransactionContext.empty(); String fileName = "testCreateFile.db"; byte[] metadata = new byte[1024]; @@ -62,7 +63,7 @@ public void testCreateFile() throws IOException, FileException, PageException { /** * 进行插入的测试 */ - public void testInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -79,7 +80,7 @@ public void testInsert() throws FileException, IOException, PageException, UUIDE /** * 对插入一个已分配UUID的测试 */ - public void testInsertWithUUID() throws FileException, IOException, PageException { + public void testInsertWithUUID() throws FileException, IOException, PageException, LogException { String fileName = "testInsertWithUUID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -88,7 +89,7 @@ public void testInsertWithUUID() throws FileException, IOException, PageExceptio long uuid = (s << 32) + rnd; TransactionContext txContext = TransactionContext.empty(); // 第一次插入，表中没有该UUID，可以正常执行 - iItemStorage.insertItemWithUuid(txContext, bytes, uuid); + iItemStorage.insertItemWithUuid(bytes, uuid); try { assertArrayEquals(bytes, iItemStorage.queryItemByUuid(uuid)); } catch (UUIDException | PageCorruptedException e) { @@ -96,14 +97,14 @@ public void testInsertWithUUID() throws FileException, IOException, PageExceptio } // 第二次插入 - iItemStorage.insertItemWithUuid(txContext, bytes, uuid); + iItemStorage.insertItemWithUuid(bytes, uuid); } /** * 对插入大量数据进行测试 */ - public void testManyInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testManyInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = "testInsertMany.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -123,7 +124,7 @@ public void testManyInsert() throws FileException, IOException, PageException, U /** * 获取整个页的数据项进行测试 */ - public void testQueryByPageID() throws FileException, IOException, PageException, PageCorruptedException { + public void testQueryByPageID() throws FileException, IOException, PageException, PageCorruptedException, LogException { String fileName = "testQueryByPageID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -186,7 +187,7 @@ public void run() { /** * 测试并发下插入是否有问题 */ - public void testSynInsert() throws IOException, FileException, PageException { + public void testSynInsert() throws IOException, FileException, PageException, LogException { String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -201,7 +202,7 @@ public void testSynInsert() throws IOException, FileException, PageException { /** * 对更新进行测试 */ - public void testUpdate() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testUpdate() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = "testUpdate.db"; TransactionContext txContext = TransactionContext.empty(); IItemStorage iItemStorage = ItemManager.fromFile(fileName); @@ -228,7 +229,7 @@ public void testUpdate() throws FileException, IOException, PageException, UUIDE /** * 测试修改和获取表头信息 */ - public void testMeta() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testMeta() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = "testMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] result = new byte[]{1, 2, 3, 4}; From c18e172525bf5444af1baa43ad513754c795c25a Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 12:29:00 +0800 Subject: [PATCH 10/20] =?UTF-8?q?=E6=97=A5=E5=BF=97=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 27 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 65 +++- .../dataitem/mock/MockItemStorage.java | 24 +- .../kaaass/rumbase/record/IRecordStorage.java | 4 +- .../rumbase/record/MvccRecordStorage.java | 4 +- .../rumbase/recovery/IRecoveryStorage.java | 12 +- .../rumbase/recovery/RecoveryManager.java | 4 +- .../rumbase/recovery/RecoveryStorage.java | 319 +++++++++++++++--- .../recovery/exception/LogException.java | 1 + .../recovery/mock/MockRecoveryStorage.java | 16 +- .../rumbase/dataitem/IItemStorageTest.java | 2 +- .../rumbase/record/IRecordStorageTest.java | 4 +- .../rumbase/recovery/IRecoveryTest.java | 20 +- 13 files changed, 426 insertions(+), 76 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 3c90ec2..161e96a 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -3,8 +3,11 @@ import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.IRecoveryStorage; import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.List; /** @@ -14,6 +17,20 @@ */ public interface IItemStorage { + void setMetaUuid(long uuid) throws IOException, PageException; + + /** + * 获得日志管理器 + * @return + */ + IRecoveryStorage getRecoveryStorage(); + + /** + * 获取表的tempFreePage + * @return + */ + public int getMaxPageId(); + /** * 将uuid对应的页强制写回 */ @@ -25,7 +42,7 @@ public interface IItemStorage { * @param item 数据项 * @return 返回数据项的UUID */ - long insertItem(TransactionContext txContext, byte[] item); + long insertItem(TransactionContext txContext, byte[] item) throws IOException, FileException; /** * 不用日志进行插入，用于日志的管理 @@ -71,7 +88,9 @@ public interface IItemStorage { * @param item 数据项 * @throws UUIDException 没有找到对应UUID的异常 */ - void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException; + void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, FileException, IOException; + + byte[] updateItemWithoutLog(long uuid,byte[] item) throws UUIDException; /** * 获得数据项存储的元数据（可以用于头） @@ -85,13 +104,13 @@ public interface IItemStorage { * * @param metadata 头信息 */ - void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException; + void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException; /** * 不使用日志设置元数据 * @param metadata 头信息 */ - void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException; + long setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException; /** * 清理多余的数据项，空间清理时使用。 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index e2e023e..21d9d5a 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -21,7 +21,6 @@ import java.util.List; import java.util.Optional; import java.util.Random; -import java.util.logging.LogManager; /** * 数据项管理器的具体实现 @@ -111,7 +110,7 @@ public static IItemStorage ofFile(String fileName) throws FileException, PageExc header.pin(); if (checkTableHeader(header)) { // 如果表头标志存在，就解析对应表头信息 - var h = parseHeader(header); + var h = parseTableHeader(header); header.unpin(); var logStorage = RecoveryManager.getRecoveryStorage(fileName); return new ItemStorage(fileName, h.tempFreePage, h.headerUuid, pageStorage,logStorage); @@ -144,7 +143,7 @@ public static IItemStorage ofFileWithoutLog(String fileName) throws FileExceptio header.pin(); if (checkTableHeader(header)) { // 如果表头标志存在，就解析对应表头信息 - var h = parseHeader(header); + var h = parseTableHeader(header); header.unpin(); return new ItemStorage(fileName, h.tempFreePage, h.headerUuid, pageStorage); } else { @@ -228,7 +227,7 @@ private void releasePage(Page page) { * * @return 解析得到的表头对象 */ - private static TableHeader parseHeader(Page page) throws PageCorruptedException { + private static TableHeader parseTableHeader(Page page) throws PageCorruptedException { try { return JBBPParser.prepare("int headerFlag;int tempFreePage;byte hasHeaderInfo;long headerUuid;"). parse(page.getData()).mapTo(new TableHeader()); @@ -328,6 +327,32 @@ private void addTempFreePage() { } } + @Override + public void setMetaUuid(long uuid) throws IOException, PageException { + var page = getPage(0); + try { + var bytes = JBBPOut.BeginBin(). + Byte(HAS_HEADER). + Long(uuid).End().toByteArray(); + page.patchData(HEADER_OFFSET, bytes); + }finally { + releasePage(page); + } + + } + + @Override + public IRecoveryStorage getRecoveryStorage() { + return this.recoveryStorage; + } + + @Override + public int getMaxPageId() { + var page = getPage(0); + var header = parseTableHeader(page); + return header.tempFreePage; + } + @Override public void flush(long uuid) throws FileException { var page = getPage(uuid); @@ -335,8 +360,9 @@ public void flush(long uuid) throws FileException { } @Override - public synchronized long insertItem(TransactionContext txContext, byte[] item) { + public synchronized long insertItem(TransactionContext txContext, byte[] item) throws IOException, FileException { long uuid = insertItemWithoutLog(item); + recoveryStorage.insert(txContext.getXid(),uuid,item); return uuid; } @Override @@ -352,7 +378,7 @@ public synchronized long insertItemWithoutLog(byte[] item) { if (pageHeader.leftSpace - Math.min(item.length, MAX_RECORD_SIZE) <= MIN_LEFT_SPACE) { // 如果剩余空间过小的话，就切换到下一个页进行，同时修改表头信息.并且，若数据过大则使用拉链，所以取512和数据大小较小的 addTempFreePage(); - return insertItemWithoutLog( item); + return insertItemWithoutLog(item); } else { // 剩余空间足够，则插入 int rnd = Math.abs(new Random().nextInt()); @@ -524,15 +550,23 @@ public List listItemByPageId(int pageId) { } @Override - public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException { + public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, FileException, IOException { + var item_before = updateItemWithoutLog(uuid, item); + recoveryStorage.update(txContext.getXid(),uuid,item_before,item); + } + + @Override + public byte[] updateItemWithoutLog(long uuid, byte[] item) throws UUIDException { var page = getPage(uuid); try { if (checkUuidExist(uuid)) { + // 若uuid存在 var pageHeader = getPageHeader(page); var items = pageHeader.get().item; try { for (var i : items) { if (i.uuid == getRndByUuid(uuid)) { + var item_before = parseData(page,i); var offset = i.offset; var bytes = JBBPOut.BeginBin(). Byte(NORMAL_DATA). @@ -540,24 +574,26 @@ public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] ite .Byte(item) .End().toByteArray(); page.patchData(offset, bytes); - return; + return item_before; } } } catch (PageException | IOException e) { throw new PageCorruptedException(2, e); } } else { + // uuid不存在一般是恢复的时候 前面事务被取消. throw new UUIDException(2); } } finally { releasePage(page); } + return new byte[1]; } @Override public byte[] getMetadata() { var page = getPage(0); - var header = parseHeader(page); + var header = parseTableHeader(page); if (checkTableHeader(page) && header.hasHeaderInfo == HAS_HEADER) { // 若表头已经被初始化并且有标志位的话，就说明有表头信息，进行获取. byte[] h; @@ -578,13 +614,15 @@ public byte[] getMetadata() { } @Override - public void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException { - setMetadataWithoutLog(metadata); - + public void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException { + var page = getPage(0); + var header = parseTableHeader(page); + var uuid = setMetadataWithoutLog(metadata); + recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); } @Override - public void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { + public long setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { var page = getPage(0); try { var headerUuid = insertItemWithoutLog(metadata); @@ -592,6 +630,7 @@ public void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException Byte(HAS_HEADER). Long(headerUuid).End().toByteArray(); page.patchData(HEADER_OFFSET, bytes); + return headerUuid; } catch (Exception e) { throw new PageCorruptedException(1, e); } finally { diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index 3ffe357..223d691 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -4,6 +4,7 @@ import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.recovery.IRecoveryStorage; import net.kaaass.rumbase.transaction.TransactionContext; import java.util.*; @@ -80,6 +81,21 @@ public static IItemStorage ofNewFile(TransactionContext txContext, String fileNa } + @Override + public void setMetaUuid(long uuid) { + + } + + @Override + public IRecoveryStorage getRecoveryStorage() { + return null; + } + + @Override + public int getMaxPageId() { + return 0; + } + @Override public void flush(long uuid) { @@ -126,6 +142,11 @@ public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] ite } } + @Override + public byte[] updateItemWithoutLog(long uuid, byte[] item) throws UUIDException { + return new byte[0]; + } + @Override public byte[] getMetadata() { return meta; @@ -137,8 +158,9 @@ public void setMetadata(TransactionContext txContext, byte[] metadata) { } @Override - public void setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { + public long setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException { + return 0; } diff --git a/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java b/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java index fb751a0..2ee1022 100644 --- a/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java +++ b/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java @@ -1,8 +1,10 @@ package net.kaaass.rumbase.record; +import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.record.exception.RecordNotFoundException; import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.Optional; /** @@ -63,5 +65,5 @@ default byte[] query(TransactionContext txContext, long recordId) throws RecordN * * @param metadata 元信息数据 */ - void setMetadata(TransactionContext txContext, byte[] metadata); + void setMetadata(TransactionContext txContext, byte[] metadata) throws IOException, FileException; } diff --git a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java index a3b1a28..ffb4551 100644 --- a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java +++ b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java @@ -6,6 +6,7 @@ import lombok.extern.slf4j.Slf4j; import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.record.exception.NeedRollbackException; import net.kaaass.rumbase.record.exception.RecordNotFoundException; import net.kaaass.rumbase.record.exception.StorageCorruptedException; @@ -13,6 +14,7 @@ import net.kaaass.rumbase.transaction.TransactionIsolation; import net.kaaass.rumbase.transaction.TransactionStatus; +import java.io.IOException; import java.util.Optional; /** @@ -205,7 +207,7 @@ public byte[] getMetadata(TransactionContext txContext) { } @Override - public void setMetadata(TransactionContext txContext, byte[] metadata) { + public void setMetadata(TransactionContext txContext, byte[] metadata) throws IOException, FileException { // TODO 申请全表锁，用this锁代替 // 主要逻辑：为了保证事务性，使用UUID数组，结合可见性选择对应的记录。倒序记录便于存储。 synchronized (this) { diff --git a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java index 372e8a3..5d2585b 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java @@ -1,6 +1,8 @@ package net.kaaass.rumbase.recovery; import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; import java.io.IOException; import java.util.List; @@ -48,18 +50,22 @@ public interface IRecoveryStorage { * @param xid * @param uuid */ - void update(int xid, long uuid, byte[] item_before,byte[] item_after) throws IOException, FileException; + void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws IOException, FileException; /** * 更新数据项的日志头 * * @param xid - * @param metaUUID 头信息的UUID */ - void updateMeta(int xid, long metaUUID) throws IOException, FileException; + void updateMeta(int xid, long beforeUuid,byte[] metadata) throws IOException, FileException; /** * 模拟打印日志资料 */ List getContent(); + + /** + * 恢复数据 + */ + void recovery() throws IOException, LogException, FileException, PageException; } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index 3957bca..8385d08 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -25,10 +25,10 @@ public static void recovery(String fileName) { } public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException, PageException { - return RecoveryStorage.ofFile(fileName + ".log"); + return RecoveryStorage.ofFile(fileName); } public static IRecoveryStorage createRecoveryStorage(String fileName) throws IOException, FileException, PageException, LogException { - return RecoveryStorage.ofNewFile(fileName + ".log"); + return RecoveryStorage.ofNewFile(fileName); } } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index 7ca44cd..cfea86d 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -5,16 +5,16 @@ import com.igormaznitsa.jbbp.mapper.Bin; import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.ItemManager; -import net.kaaass.rumbase.page.PageManager; -import net.kaaass.rumbase.page.PageStorage; +import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; -import net.kaaass.rumbase.transaction.TransactionContext; -import net.kaaass.rumbase.transaction.mock.MockTransactionContext; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; /** * 日志记录相关内容 @@ -23,10 +23,19 @@ */ public class RecoveryStorage implements IRecoveryStorage { - private IItemStorage itemStorage; - public RecoveryStorage(IItemStorage itemStorage) { - this.itemStorage = itemStorage; + /** + * 用于对自己的日志记录进行管理 + */ + private IItemStorage logStorage; + /** + * 要恢复的对象对应的文件名 + */ + private String fileName; + + public RecoveryStorage(IItemStorage itemStorage,String fileName) { + this.logStorage = itemStorage; + this.fileName = fileName; } /** @@ -36,7 +45,7 @@ public RecoveryStorage(IItemStorage itemStorage) { */ public static IRecoveryStorage ofFile(String fileName) throws FileException, IOException, LogException, PageException { var itemStorage = ItemManager.fromFileWithoutLog(fileName + ".log"); - return new RecoveryStorage(itemStorage); + return new RecoveryStorage(itemStorage,fileName); } /** @@ -47,87 +56,237 @@ public static IRecoveryStorage ofFile(String fileName) throws FileException, IOE public static IRecoveryStorage ofNewFile(String fileName) throws FileException, IOException, PageException, LogException { var metadata = JBBPOut.BeginBin().Int(HEADER).End().toByteArray(); var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log",metadata); - return new RecoveryStorage(itemStorage); + return new RecoveryStorage(itemStorage,fileName); } @Override public void begin(int xid, List snapshots) throws IOException, FileException { var jbbp = JBBPOut.BeginBin(). - String(TX_BEGIN). + Byte(TX_BEGIN). Int(xid). Int(snapshots.size()); for ( var i : snapshots){ jbbp = jbbp.Int(i); } var bytes = jbbp.End().toByteArray(); - var uuid = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(uuid); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); } @Override public void rollback(int xid) throws IOException, FileException { var bytes = JBBPOut.BeginBin(). - String(TX_ABORT). + Byte(TX_ABORT). Int(xid). End().toByteArray(); - var uuid = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(uuid); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); } @Override public void commit(int xid) throws IOException, FileException { var bytes = JBBPOut.BeginBin(). - String(TX_COMMIT). + Byte(TX_COMMIT). Int(xid). End().toByteArray(); - var uuid = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(uuid); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); } @Override public void insert(int xid, long uuid, byte[] item) throws IOException, FileException { var bytes = JBBPOut.BeginBin(). - String(INSERT_FLAG). + Byte(INSERT_FLAG). Int(xid). Long(uuid). Int(item.length). Byte(item). End().toByteArray(); - var id = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(id); - + var id = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(id); } @Override - public void update(int xid, long uuid, byte[] item_before,byte[] item_after) throws IOException, FileException { + public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws IOException, FileException { var bytes = JBBPOut.BeginBin(). - String(UPDATE_FLAG). + Byte(UPDATE_FLAG). Int(xid). Long(uuid). - Int(item_before.length). - Byte(item_before). - Int(item_after.length). - Byte(item_after). + Int(itemBefore.length). + Byte(itemBefore). + Int(itemAfter.length). + Byte(itemAfter). End().toByteArray(); - var id = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(id); + var id = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(id); } @Override - public void updateMeta(int xid, long metaUUID) throws IOException, FileException { + public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws IOException, FileException { var bytes =JBBPOut.BeginBin(). - String(METADATA_UPDATE_FLAG). + Byte(METADATA_UPDATE_FLAG). Int(xid). - Long(metaUUID). + Long(beforeUuid). + Int(metadata.length). + Byte(metadata). End().toByteArray(); - var uuid = itemStorage.insertItemWithoutLog(bytes); - itemStorage.flush(uuid); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); } @Override public List getContent() { - return null; + List logList = new ArrayList<>(); + var maxPageId = logStorage.getMaxPageId(); + for (int i = 1;i <= maxPageId ; i ++){ + var logs = logStorage.listItemByPageId(i); + for(var l : logs){ + if (l.length > 4){ + logList.add(l); + } + } + } + return logList; + } + + @Override + public void recovery() throws IOException, LogException, FileException, PageException { + var itemStorage = ItemManager.fromFile(this.fileName); + var logs = getContent(); + var xidMaps = parseXid(logs); + for (var log : logs){ + parseLog(log,itemStorage,xidMaps); + } + } + + /** + * 先进行一遍解析，得到所有已完成的事务和未完成的事务，来决定redo还是undo + */ + private Map> parseXid(List logs) throws IOException { + List commitXids = new ArrayList<>(); + List abortXids = new ArrayList<>(); + + for (var log : logs){ + var tx = parseTx(log); + switch (tx.type){ + case TX_COMMIT: + // 若是commit 则放入commitXid并将对应的xid移出abort + abortXids.remove(tx.xid); + commitXids.add(tx.xid); + break; + case TX_ABORT: + // 因为begin就放入abort，所以abort不用管 + break; + case TX_BEGIN: + abortXids.add(tx.xid); + break; + default: + } + } + + Map> maps = new HashMap<>(); + maps.put('C',commitXids); + maps.put('A',abortXids); + return maps; + } + + /** + * 解析事务状态 + */ + private Tx parseTx(byte[] log) throws IOException { + var tx = JBBPParser.prepare("byte type;int xid;").parse(log).mapTo(new Tx()); + return tx; + } + + /** + * 解析类型 + */ + private byte parseType(byte[] log) throws IOException { + var type = JBBPParser.prepare("byte type;").parse(log).mapTo(new Type()).type; + return type ; + } + + /** + * 解析插入 + */ + private InsertLog parseInsert(byte[] log) throws IOException { + var insertLog = JBBPParser.prepare("byte type;int xid;long uuid;int length;byte[length] item;") + .parse(log).mapTo(new InsertLog()); + return insertLog; + } + + /** + * 解析更新 + */ + private UpdateLog parseUpdate(byte[] log) throws IOException { + var updateLog = JBBPParser.prepare("byte type;int xid;long uuid;int length1;" + + "byte[length1] itemBefore;int length2;byte[length2] itemAfter;").parse(log).mapTo(new UpdateLog()); + return updateLog; + } + + private UpdateMetaLog parseUpdateMeta(byte[] log) throws IOException { + var updateMeta = JBBPParser.prepare("byte type;int xid;long beforeUuid;" + + "int length;byte[length] metadata"). + parse(log).mapTo(new UpdateMetaLog()); + return updateMeta; + } + + private boolean checkCommit(int xid,Map> maps) throws LogException { + if (maps.get('C').contains(xid)){ + return true; + }else if (maps.get('A').contains(xid)){ + return false; + }else { + throw new LogException(2); + } + } + private void parseLog(byte[] log, IItemStorage itemStorage, Map> xidMaps) throws IOException, LogException, PageException { + var type = parseType(log); + switch (type){ + case INSERT_FLAG: + System.out.println("正在解析插入"); + var insertLog = parseInsert(log); + if (checkCommit(insertLog.xid,xidMaps)){ + // 如果事务已经提交，则redo + itemStorage.insertItemWithUuid(insertLog.item,insertLog.uuid); + }else { + // 如果事务没有提交，则undo + } + break; + case UPDATE_FLAG: + System.out.println("正在解析更新"); + var updateLog = parseUpdate(log); + if (checkCommit(updateLog.xid,xidMaps)){ + // 若事务已经提交则redo + try { + itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemAfter); + } catch (UUIDException ignored) { + // uuid不存在说明对应之前的事务没有执行，是正常 + } + }else { + // 若事务没有提交，则undo,恢复之前的数据 + try { + itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemBefore); + } catch (UUIDException ignored) { + + } + } + break; + case METADATA_UPDATE_FLAG: + System.out.println("正在解析头信息更新"); + var updateMetaLog = parseUpdateMeta(log); + if (checkCommit(updateMetaLog.xid,xidMaps)){ + // redo + itemStorage.setMetadataWithoutLog(updateMetaLog.metadata); + }else { + // undo + itemStorage.setMetaUuid(updateMetaLog.beforeUuid); + } + break; + default: + return; + } } /** @@ -139,22 +298,98 @@ public static class LogHeader{ int header; } + /** + * 解析类型 + */ + public static class Type{ + @Bin + byte type; + } + + /** + * 事务状态的解析 + */ + public static class Tx{ + @Bin + byte type; + @Bin + int xid; + } + + /** + * 插入的解析 + */ + public static class InsertLog{ + @Bin + byte type; + @Bin + int xid; + @Bin + long uuid; + @Bin + int length; + @Bin + byte[] item; + public Object newInstance(Class klazz) { + return klazz == InsertLog.class ? new InsertLog() : null; + } + } + + /** + * 更新的解析 + */ + public static class UpdateLog{ + @Bin + Byte type; + @Bin + int xid; + @Bin + long uuid; + @Bin + int length1; + @Bin + byte[] itemBefore; + @Bin + int length2; + @Bin + byte[] itemAfter; + public Object newInstance(Class klazz) { + return klazz == UpdateLog.class ? new UpdateLog() : null; + } + } + + public static class UpdateMetaLog{ + @Bin + Byte type; + @Bin + int xid; + @Bin + long beforeUuid; + @Bin + int length; + @Bin + byte[] metadata; + public Object newInstance(Class klazz) { + return klazz == UpdateMetaLog.class ? new UpdateMetaLog() : null; + } + } + /** * 日志标志 */ final private static int HEADER = 4567; - final private static String INSERT_FLAG = "i"; + final private static byte INSERT_FLAG = 'i'; - final private static String TX_BEGIN = "b"; + final private static byte TX_BEGIN = 'b'; - final private static String TX_ABORT = "a"; + final private static byte TX_ABORT = 'a'; - final private static String TX_COMMIT = "c"; + final private static byte TX_COMMIT = 'c'; - final private static String UPDATE_FLAG = "u"; + final private static byte UPDATE_FLAG = 'u'; - final private static String METADATA_UPDATE_FLAG = "m"; + final private static byte METADATA_UPDATE_FLAG = 'm'; diff --git a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java index d15b0a5..e22f074 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java +++ b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java @@ -13,6 +13,7 @@ public class LogException extends RumbaseException { public static final Map REASONS = new HashMap() {{ put(1, "日志文件无法解析"); + put(2,"事务ID不存在"); }}; public LogException(int subId) { diff --git a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java index 461cf2e..b872dbe 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java @@ -2,6 +2,7 @@ import net.kaaass.rumbase.recovery.IRecoveryStorage; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -49,19 +50,17 @@ public void insert(int xid, long uuid, byte[] item) { } @Override - public void update(int xid, long uuid, byte[] item,byte[] item_after) { + public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) { String updateStr = "update " + xid + " " + uuid + " "; // 对控制语句、数据两部分进行合并得到最终日志记录 byte[] first = updateStr.getBytes(); - byte[] result = Arrays.copyOf(first, first.length + item.length); - System.arraycopy(item, 0, result, first.length, item.length); + byte[] result = Arrays.copyOf(first, first.length + itemBefore.length); + System.arraycopy(itemBefore, 0, result, first.length, itemBefore.length); bytes.add(result); } @Override - public void updateMeta(int xid, long metaUUID) { - String updateMetaStr = "meta " + xid + " " + metaUUID; - bytes.add(updateMetaStr.getBytes()); + public void updateMeta(int xid,long beforeUuid,byte[] metadata) { } @Override @@ -69,5 +68,10 @@ public List getContent() { return bytes; } + @Override + public void recovery() throws IOException { + + } + } diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index ca0bb4c..8f8fb91 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -109,7 +109,7 @@ public void testManyInsert() throws FileException, IOException, PageException, U IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; TransactionContext txContext = TransactionContext.empty(); - for (int i = 0; i < 1000; i++) { + for (int i = 0; i < 500; i++) { long uuid = iItemStorage.insertItem(txContext, bytes); long uuid2 = iItemStorage.insertItem(txContext, bytes); long uuid3 = iItemStorage.insertItem(txContext, bytes); diff --git a/src/test/java/net/kaaass/rumbase/record/IRecordStorageTest.java b/src/test/java/net/kaaass/rumbase/record/IRecordStorageTest.java index f7d9ed5..0fe2f8b 100644 --- a/src/test/java/net/kaaass/rumbase/record/IRecordStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/record/IRecordStorageTest.java @@ -2,9 +2,11 @@ import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; +import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.record.exception.RecordNotFoundException; import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.UUID; import static org.junit.Assert.assertArrayEquals; @@ -57,7 +59,7 @@ public void testDelete() throws RecordNotFoundException { assertTrue("record should be deleted", result.isEmpty()); } - public void testMetadata() { + public void testMetadata() throws IOException, FileException { var storage = RecordManager.fromFile("test_metadata"); var context = TransactionContext.empty(); diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index 118b2d2..425a4bc 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -2,7 +2,15 @@ import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; +import net.kaaass.rumbase.dataitem.IItemStorage; +import net.kaaass.rumbase.dataitem.ItemManager; +import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.FileException; +import net.kaaass.rumbase.page.exception.PageException; +import net.kaaass.rumbase.recovery.exception.LogException; +import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -14,7 +22,17 @@ @Slf4j public class IRecoveryTest extends TestCase { - public void testBegin() { + public void testBegin() throws PageException, LogException, FileException, IOException, UUIDException { + String fileName = "testInsert.db"; + IItemStorage iItemStorage = ItemManager.fromFile(fileName); + byte[] bytes = new byte[]{1, 2, 3, 4}; + TransactionContext txContext = TransactionContext.empty(); + long uuid = iItemStorage.insertItem(txContext, bytes); + assertArrayEquals(bytes, iItemStorage.queryItemByUuid(uuid)); + + var recoveryStorage = iItemStorage.getRecoveryStorage(); + var logs = recoveryStorage.getContent(); + recoveryStorage.recovery(); } } From bf377d12388609d94cf645b2ad6ecbe1a0c78d40 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 15:34:42 +0800 Subject: [PATCH 11/20] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 2 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 3 +- .../dataitem/mock/MockItemStorage.java | 3 +- .../rumbase/recovery/RecoveryStorage.java | 9 +- .../rumbase/recovery/IRecoveryTest.java | 130 +++++++++++++++++- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 161e96a..3c98f32 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -104,7 +104,7 @@ public interface IItemStorage { * * @param metadata 头信息 */ - void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException; + long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException; /** * 不使用日志设置元数据 diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index 21d9d5a..7275833 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -614,11 +614,12 @@ public byte[] getMetadata() { } @Override - public void setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException { + public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException { var page = getPage(0); var header = parseTableHeader(page); var uuid = setMetadataWithoutLog(metadata); recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); + return uuid; } @Override diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index 223d691..241f2ab 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -153,8 +153,9 @@ public byte[] getMetadata() { } @Override - public void setMetadata(TransactionContext txContext, byte[] metadata) { + public long setMetadata(TransactionContext txContext, byte[] metadata) { this.meta = metadata; + return 0; } @Override diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index cfea86d..446a854 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -172,7 +172,8 @@ private Map> parseXid(List logs) throws IOExcept switch (tx.type){ case TX_COMMIT: // 若是commit 则放入commitXid并将对应的xid移出abort - abortXids.remove(tx.xid); + Integer id = tx.xid; + abortXids.remove(id); commitXids.add(tx.xid); break; case TX_ABORT: @@ -227,7 +228,7 @@ private UpdateLog parseUpdate(byte[] log) throws IOException { private UpdateMetaLog parseUpdateMeta(byte[] log) throws IOException { var updateMeta = JBBPParser.prepare("byte type;int xid;long beforeUuid;" + - "int length;byte[length] metadata"). + "int length;byte[length] metadata;"). parse(log).mapTo(new UpdateMetaLog()); return updateMeta; } @@ -340,7 +341,7 @@ public Object newInstance(Class klazz) { */ public static class UpdateLog{ @Bin - Byte type; + byte type; @Bin int xid; @Bin @@ -360,7 +361,7 @@ public Object newInstance(Class klazz) { public static class UpdateMetaLog{ @Bin - Byte type; + byte type; @Bin int xid; @Bin diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index 425a4bc..a569c15 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -12,7 +12,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Random; import static org.junit.Assert.assertArrayEquals; @@ -22,17 +24,135 @@ @Slf4j public class IRecoveryTest extends TestCase { - public void testBegin() throws PageException, LogException, FileException, IOException, UUIDException { + public void testInsert() throws PageException, LogException, FileException, IOException, UUIDException { String fileName = "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - TransactionContext txContext = TransactionContext.empty(); - long uuid = iItemStorage.insertItem(txContext, bytes); + var txContext = TransactionContext.empty(); + var recoveryStorage = iItemStorage.getRecoveryStorage(); + var xid = 1; + var xid2 = 0; + List snaps = new ArrayList<>(); + // 测试日志中存在，但是表中不存在进行恢复 + recoveryStorage.begin(xid,snaps); + long a = 1; + long uuid = (a << 32) + Math.abs(new Random().nextInt()); + recoveryStorage.insert(xid,uuid,bytes); + recoveryStorage.commit(xid); + + // 测试日志中存在，且数据中也存在，会不会重复插入 + + recoveryStorage.begin(xid2,snaps); + iItemStorage.insertItem(txContext,bytes); + recoveryStorage.commit(xid2); - assertArrayEquals(bytes, iItemStorage.queryItemByUuid(uuid)); + recoveryStorage.recovery(); + // 测试日志有额外的数据有没有被插入 + var result = iItemStorage.queryItemByUuid(uuid); + assertTrue(Arrays.equals(result,bytes)); + // 测试表中有的数据是否被重复插入 + var list = iItemStorage.listItemByPageId(1); + assertEquals(2,list.size()); + } + public void testUpdate() throws PageException, LogException, FileException, IOException, UUIDException { + String fileName = "testUpdate.db"; + IItemStorage iItemStorage = ItemManager.fromFile(fileName); + byte[] bytes = new byte[]{1, 2, 3, 4}; + byte[] bytesUpdate = new byte[]{2,3,4,5}; + var txContext = TransactionContext.empty(); var recoveryStorage = iItemStorage.getRecoveryStorage(); - var logs = recoveryStorage.getContent(); + + int xid = 0; + List snaps = new ArrayList<>(); + + // 测试在表中存在的redo + recoveryStorage.begin(xid,snaps); + long uuid = iItemStorage.insertItem(txContext,bytes); + iItemStorage.updateItemByUuid(txContext,uuid,bytesUpdate); + recoveryStorage.commit(xid); + + // 测试在表中不存在的redo + int xid2 = 1; + recoveryStorage.begin(xid2,snaps); + long a = 1; + long uuid2 = (a << 32) + Math.abs(new Random().nextInt()); + recoveryStorage.insert(xid2,uuid2,bytes); + recoveryStorage.update(xid2,uuid2,bytes,bytesUpdate); + recoveryStorage.commit(xid2); + + // 测试abort的事务 + int xid3 = 2; + recoveryStorage.begin(xid3,snaps); + long b = 1; + long uuid3 = (b << 32) + Math.abs(new Random().nextInt()); + recoveryStorage.insert(xid3,uuid3,bytes); + recoveryStorage.commit(xid3); + int xid4 = 3; + recoveryStorage.begin(xid4,snaps); + recoveryStorage.update(xid4,uuid3,bytes,bytesUpdate); + recoveryStorage.rollback(xid4); + recoveryStorage.recovery(); + + assertTrue(Arrays.equals(bytesUpdate,iItemStorage.queryItemByUuid(uuid))); + assertTrue(Arrays.equals(bytesUpdate,iItemStorage.queryItemByUuid(uuid2))); + assertTrue(Arrays.equals(bytes,iItemStorage.queryItemByUuid(uuid3))); + var list = iItemStorage.listItemByPageId(1); + assertEquals(3,list.size()); } + + public void testUpdateMeta() throws PageException, LogException, FileException, IOException { + String fileName = "testUpdateMeta.db"; + IItemStorage iItemStorage = ItemManager.fromFile(fileName); + byte[] bytes = new byte[]{1, 2, 3, 4}; + byte[] bytesUpdate = new byte[]{2,3,4,5}; + var txContext = TransactionContext.empty(); + var recoveryStorage = iItemStorage.getRecoveryStorage(); + + // 测试表中有 + int xid = 0; + List snaps = new ArrayList<>(); + recoveryStorage.begin(xid,snaps); + var uuid = iItemStorage.setMetadata(txContext,bytes); + recoveryStorage.commit(xid); + + assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); + // 测试表中没有头信息时更新 + int xid2 = 1; + recoveryStorage.begin(xid2,snaps); + recoveryStorage.updateMeta(xid2,uuid,bytesUpdate); + recoveryStorage.commit(xid2); + recoveryStorage.recovery(); + assertTrue(Arrays.equals(bytesUpdate,iItemStorage.getMetadata())); + + } + + public void testUpdateMetaFail() throws PageException, LogException, FileException, IOException { + String fileName = "testUpdateMetaFail.db"; + IItemStorage iItemStorage = ItemManager.fromFile(fileName); + byte[] bytes = new byte[]{1, 2, 3, 4}; + byte[] bytesUpdate = new byte[]{2,3,4,5}; + var txContext = TransactionContext.empty(); + var recoveryStorage = iItemStorage.getRecoveryStorage(); + + // 测试表中有 + int xid = 0; + List snaps = new ArrayList<>(); + recoveryStorage.begin(xid,snaps); + var uuid = iItemStorage.setMetadata(txContext,bytes); + recoveryStorage.commit(xid); + + assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); + // 测试事务失败 + int xid2 = 1; + recoveryStorage.begin(xid2,snaps); + recoveryStorage.updateMeta(xid2,uuid,bytesUpdate); + recoveryStorage.rollback(xid2); + recoveryStorage.recovery(); + assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); + + } + + } From 30a70aafbf970718de1466f3f59880b456bd1083 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 15:58:02 +0800 Subject: [PATCH 12/20] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=8F=92=E5=85=A5?= =?UTF-8?q?=E7=9A=84=E5=9B=9E=E6=BB=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 6 ++ .../kaaass/rumbase/dataitem/ItemStorage.java | 57 ++++++++++++++++--- .../dataitem/mock/MockItemStorage.java | 7 +++ .../rumbase/recovery/RecoveryStorage.java | 1 + .../rumbase/recovery/IRecoveryTest.java | 21 +++++++ 5 files changed, 84 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index 3c98f32..b829d7f 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -118,6 +118,12 @@ public interface IItemStorage { * @param uuids 数据项UUID的编号列表 */ void removeItems(List uuids); + + /** + * 删除对应的uuid + * @param uuid + */ + void deleteUuid(long uuid) throws IOException, PageException; } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index 7275833..e23c548 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -349,13 +349,18 @@ public IRecoveryStorage getRecoveryStorage() { @Override public int getMaxPageId() { var page = getPage(0); - var header = parseTableHeader(page); - return header.tempFreePage; + try{ + var header = parseTableHeader(page); + return header.tempFreePage; + }finally { + releasePage(page); + } } @Override public void flush(long uuid) throws FileException { var page = getPage(uuid); + releasePage(page); page.flush(); } @@ -465,6 +470,33 @@ public synchronized void insertItemWithUuid(byte[] item, long uuid) { // 若存在则不需要恢复，直接返回 } + /** + * 日志恢复时用，回退对应的insert操作 + * @param uuid + */ + @Override + public void deleteUuid(long uuid) throws IOException, PageException { + var page = getPage(uuid); + try { + var rnd = getRndByUuid(uuid); + var header = getPageHeader(page); + if (header.isPresent()){ + var items = header.get().item; + for (int i = 0;i < items.length ; i ++){ + if (rnd == items[i].uuid){ + // 该item对应的起止位置,将其数据项变为-1 + int offset = ITEM_OFFSET + i * ITEM_SIZE; + var bytes = JBBPOut.BeginBin().Int(-1).Int(0).End().toByteArray(); + page.patchData(offset,bytes); + return; + } + } + } + }finally { + releasePage(page); + } + } + /** * 检查uuid是否存在,若Uuid的页号超过当前可用页，则直接返回False * @@ -537,8 +569,12 @@ public List listItemByPageId(int pageId) { if (pageHeaderOp.isPresent()) { var pageHeader = pageHeaderOp.get(); for (var item : pageHeader.item) { - var data = parseData(page, item); - bytes.add(data); + if (item.uuid < 0){ + continue; + }else { + var data = parseData(page, item); + bytes.add(data); + } } } return bytes; @@ -616,10 +652,15 @@ public byte[] getMetadata() { @Override public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException { var page = getPage(0); - var header = parseTableHeader(page); - var uuid = setMetadataWithoutLog(metadata); - recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); - return uuid; + try{ + var header = parseTableHeader(page); + var uuid = setMetadataWithoutLog(metadata); + recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); + return uuid; + }finally { + releasePage(page); + } + } @Override diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index 241f2ab..5010d7d 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -4,9 +4,11 @@ import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.IRecoveryStorage; import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.IOException; import java.util.*; /** @@ -170,6 +172,11 @@ public void removeItems(List uuids) { System.out.println("已经清除文件对应uuid的信息"); } + @Override + public void deleteUuid(long uuid) throws IOException, PageException { + + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index 446a854..80b7e04 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -253,6 +253,7 @@ private void parseLog(byte[] log, IItemStorage itemStorage, Map snaps = new ArrayList<>(); + recoveryStorage.begin(xid,snaps); + long uuid = iItemStorage.insertItem(txContext,bytes); + recoveryStorage.rollback(xid); + recoveryStorage.recovery(); + try { + var item = iItemStorage.queryItemByUuid(uuid); + assertFalse(Arrays.equals(bytes,item)); + }catch (Exception e){ + + } + + } + public void testUpdate() throws PageException, LogException, FileException, IOException, UUIDException { String fileName = "testUpdate.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); From 9d221a5be6d92f2a81d49b2371fc6839531fdc6b Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 15:59:07 +0800 Subject: [PATCH 13/20] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=8F=92=E5=85=A5?= =?UTF-8?q?=E7=9A=84=E5=9B=9E=E6=BB=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index c5413c9..e7ba030 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -56,7 +56,7 @@ public void testInsert() throws PageException, LogException, FileException, IOEx } public void testInsertFail() throws PageException, LogException, FileException, IOException, UUIDException { - String fileName = "testInsert.db"; + String fileName = "testInsertFailed.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; var txContext = TransactionContext.empty(); From 749c5f85158beefa3ccfa69f1a8e3033b74eb6d2 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 16:34:22 +0800 Subject: [PATCH 14/20] =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/net/kaaass/rumbase/recovery/RecoveryManager.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index 8385d08..1d7d98e 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -19,9 +19,9 @@ public class RecoveryManager { * @param fileName 文件名 * @return 数据库日志管理器 */ - public static void recovery(String fileName) { + public static void recovery(String fileName) throws PageException, LogException, FileException, IOException { // TODO:对数据进行恢复 - + RecoveryStorage.ofFile(fileName).recovery(); } public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException, PageException { From bf583151a78849476fc9bbb2be82df240acc46b0 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 16:34:35 +0800 Subject: [PATCH 15/20] =?UTF-8?q?=E5=90=AF=E5=8A=A8=E6=81=A2=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index 1d7d98e..6fb4be7 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -20,7 +20,7 @@ public class RecoveryManager { * @return 数据库日志管理器 */ public static void recovery(String fileName) throws PageException, LogException, FileException, IOException { - // TODO:对数据进行恢复 + RecoveryStorage.ofFile(fileName).recovery(); } From 0220686301fd30ca3f0f7f118840a6bb82635dbd Mon Sep 17 00:00:00 2001 From: KAAAsS Date: Sat, 16 Jan 2021 21:19:48 +0800 Subject: [PATCH 16/20] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E7=AD=BE=E5=90=8D=E5=8F=98=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/record/IRecordStorage.java | 2 +- .../rumbase/record/MvccRecordStorage.java | 20 ++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java b/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java index ea5582b..26d1e8b 100644 --- a/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java +++ b/src/main/java/net/kaaass/rumbase/record/IRecordStorage.java @@ -65,5 +65,5 @@ default byte[] query(TransactionContext txContext, long recordId) throws RecordN * * @param metadata 元信息数据 */ - void setMetadata(TransactionContext txContext, byte[] metadata) throws IOException, FileException; + void setMetadata(TransactionContext txContext, byte[] metadata); } diff --git a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java index ec12262..541d964 100644 --- a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java +++ b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java @@ -47,7 +47,13 @@ public long insert(TransactionContext txContext, byte[] rawData) { writePayload(data, rawData); // 不用检查版本跳跃的原因是，插入本身不用；更新操作必定先删除，而删除检查 // 插入记录 - return storage.insertItem(txContext, data); + try { + return storage.insertItem(txContext, data); + } catch (IOException | FileException e) { + e.printStackTrace(); + // FIXME + return -1; + } } @Override @@ -114,6 +120,9 @@ public void delete(TransactionContext txContext, long recordId) throws RecordNot storage.updateItemByUuid(txContext, recordId, data); } catch (UUIDException e) { throw new RecordNotFoundException(1, e); + } catch (IOException | FileException e) { + e.printStackTrace(); + // FIXME } } @@ -222,7 +231,7 @@ public byte[] getMetadata(TransactionContext txContext) { } @Override - public void setMetadata(TransactionContext txContext, byte[] metadata) throws IOException, FileException { + public void setMetadata(TransactionContext txContext, byte[] metadata) { // TODO 申请全表锁，用this锁代替 // 主要逻辑：为了保证事务性，使用UUID数组，结合可见性选择对应的记录。倒序记录便于存储。 synchronized (this) { @@ -233,7 +242,12 @@ public void setMetadata(TransactionContext txContext, byte[] metadata) throws IO var newMetadata = new byte[oldMetadata.length + 8]; System.arraycopy(oldMetadata, 0, newMetadata, 8, oldMetadata.length); MvccUtil.writeLong(newMetadata, 0, newId); - storage.setMetadata(txContext, newMetadata); + try { + storage.setMetadata(txContext, newMetadata); + } catch (IOException | FileException e) { + e.printStackTrace(); + // FIXME + } // 缓存取消 this.metadataCache = null; } From 67e8b873296d87731e95c186e6e9b3138788b352 Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 22:07:58 +0800 Subject: [PATCH 17/20] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rumbase/recovery/IRecoveryTest.java | 22 ++++++++++++++---- testInsert.db | Bin 0 -> 40960 bytes 2 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 testInsert.db diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index e7ba030..3197f88 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -10,6 +10,7 @@ import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; +import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; @@ -25,7 +26,9 @@ public class IRecoveryTest extends TestCase { public void testInsert() throws PageException, LogException, FileException, IOException, UUIDException { - String fileName = "testInsert.db"; + new File("test_gen_files/testInsert.db").delete(); + new File("test_gen_files/testInsert.db.log").delete(); + String fileName = "test_gen_files/testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; var txContext = TransactionContext.empty(); @@ -53,10 +56,13 @@ public void testInsert() throws PageException, LogException, FileException, IOEx // 测试表中有的数据是否被重复插入 var list = iItemStorage.listItemByPageId(1); assertEquals(2,list.size()); + } public void testInsertFail() throws PageException, LogException, FileException, IOException, UUIDException { - String fileName = "testInsertFailed.db"; + new File("test_gen_files/testInsertFailed.db").delete(); + new File("test_gen_files/testInsertFailed.db.log").delete(); + String fileName = "test_gen_files/testInsertFailed.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; var txContext = TransactionContext.empty(); @@ -77,7 +83,9 @@ public void testInsertFail() throws PageException, LogException, FileException, } public void testUpdate() throws PageException, LogException, FileException, IOException, UUIDException { - String fileName = "testUpdate.db"; + new File("test_gen_files/testUpdate.db").delete(); + new File("test_gen_files/testUpdate.db.log").delete(); + String fileName = "test_gen_files/testUpdate.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; byte[] bytesUpdate = new byte[]{2,3,4,5}; @@ -124,7 +132,9 @@ public void testUpdate() throws PageException, LogException, FileException, IOEx } public void testUpdateMeta() throws PageException, LogException, FileException, IOException { - String fileName = "testUpdateMeta.db"; + new File("test_gen_files/testUpdateMeta.db").delete(); + new File("test_gen_files/testUpdateMeta.db.log").delete(); + String fileName = "test_gen_files/testUpdateMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; byte[] bytesUpdate = new byte[]{2,3,4,5}; @@ -150,7 +160,9 @@ public void testUpdateMeta() throws PageException, LogException, FileException, } public void testUpdateMetaFail() throws PageException, LogException, FileException, IOException { - String fileName = "testUpdateMetaFail.db"; + new File("test_gen_files/testUpdateMetaFail.db").delete(); + new File("test_gen_files/testUpdateMetaFail.db.log").delete(); + String fileName = "test_gen_files/testUpdateMetaFail.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; byte[] bytesUpdate = new byte[]{2,3,4,5}; diff --git a/testInsert.db b/testInsert.db new file mode 100644 index 0000000000000000000000000000000000000000..ca5ea44f24f65b3d8acaaf654c430de8c51b16e2 GIT binary patch literal 40960 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd t0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0Rsm92L_-300961 literal 0 HcmV?d00001 From d472e36f5ddebd1f4f4f4887ee71bf92c255d00b Mon Sep 17 00:00:00 2001 From: xiaoxineryi <529086017@qq.com> Date: Sat, 16 Jan 2021 23:30:13 +0800 Subject: [PATCH 18/20] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 15 +- .../kaaass/rumbase/dataitem/ItemManager.java | 9 +- .../kaaass/rumbase/dataitem/ItemStorage.java | 33 +-- .../rumbase/recovery/IRecoveryStorage.java | 14 +- .../rumbase/recovery/RecoveryManager.java | 6 +- .../rumbase/recovery/RecoveryStorage.java | 278 ++++++++++-------- .../recovery/exception/LogException.java | 13 +- 7 files changed, 212 insertions(+), 156 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index b829d7f..fb94da3 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -5,6 +5,7 @@ import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.IRecoveryStorage; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import java.io.IOException; @@ -17,7 +18,7 @@ */ public interface IItemStorage { - void setMetaUuid(long uuid) throws IOException, PageException; + void setMetaUuid(long uuid) throws PageException, IOException; /** * 获得日志管理器 @@ -42,7 +43,7 @@ public interface IItemStorage { * @param item 数据项 * @return 返回数据项的UUID */ - long insertItem(TransactionContext txContext, byte[] item) throws IOException, FileException; + long insertItem(TransactionContext txContext, byte[] item) throws LogException,PageCorruptedException; /** * 不用日志进行插入，用于日志的管理 @@ -88,7 +89,7 @@ public interface IItemStorage { * @param item 数据项 * @throws UUIDException 没有找到对应UUID的异常 */ - void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, FileException, IOException; + void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, LogException,PageCorruptedException; byte[] updateItemWithoutLog(long uuid,byte[] item) throws UUIDException; @@ -104,13 +105,13 @@ public interface IItemStorage { * * @param metadata 头信息 */ - long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException; + long setMetadata(TransactionContext txContext, byte[] metadata) throws LogException; /** * 不使用日志设置元数据 * @param metadata 头信息 */ - long setMetadataWithoutLog(byte[] metadata) throws PageCorruptedException; + long setMetadataWithoutLog(byte[] metadata); /** * 清理多余的数据项，空间清理时使用。 @@ -120,10 +121,10 @@ public interface IItemStorage { void removeItems(List uuids); /** - * 删除对应的uuid + * 日志恢复时用，回退对应的insert操作，做法是删除对应的uuid * @param uuid */ - void deleteUuid(long uuid) throws IOException, PageException; + void deleteUuid(long uuid) throws PageException, IOException; } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index 0ce485a..8c5148e 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -1,6 +1,7 @@ package net.kaaass.rumbase.dataitem; +import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; @@ -26,7 +27,7 @@ public class ItemManager { * @param fileName 文件名 * @return 数据项管理器，用于管理数据项 */ - public static IItemStorage fromFile(String fileName) throws FileException, IOException, PageException, LogException { + public static IItemStorage fromFile(String fileName) throws FileException, PageException, LogException { if (maps.containsKey(fileName)) { return maps.get(fileName); } else { @@ -36,7 +37,7 @@ public static IItemStorage fromFile(String fileName) throws FileException, IOExc } } - public static IItemStorage fromFileWithoutLog(String fileName) throws FileException, IOException, PageException, LogException { + public static IItemStorage fromFileWithoutLog(String fileName) throws FileException,PageException, PageCorruptedException { if (maps.containsKey(fileName)) { return maps.get(fileName); } else { @@ -56,7 +57,7 @@ public static IItemStorage fromFileWithoutLog(String fileName) throws FileExcept * @return 数据项管理器 * @throws FileException 想新建的文件已经存在的异常 */ - public static IItemStorage createFile(TransactionContext txContext, String fileName, byte[] metadata) throws FileException, IOException, PageException, LogException { + public static IItemStorage createFile(TransactionContext txContext, String fileName, byte[] metadata) throws FileException, PageException, LogException { // 如果文件已经存在，那么就抛出文件已存在异常 if (maps.containsKey(fileName)) { throw new FileException(1); @@ -68,7 +69,7 @@ public static IItemStorage createFile(TransactionContext txContext, String fileN } } - public static IItemStorage createFileWithoutLog(String fileName, byte[] metadata) throws FileException, IOException, PageException, LogException { + public static IItemStorage createFileWithoutLog(String fileName, byte[] metadata) throws FileException, PageException { // 如果文件已经存在，那么就抛出文件已存在异常 if (maps.containsKey(fileName)) { throw new FileException(1); diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index e23c548..99217c7 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -87,7 +87,7 @@ public ItemStorage(String fileName, int tempFreePage, long headerUuid, PageStora * @param header 第一页的Page对象 * @return 是否是表的第一页 */ - private static boolean checkTableHeader(Page header) { + private static boolean checkTableHeader(Page header) throws PageCorruptedException{ var data = header.getData(); byte[] flag = new byte[4]; try { @@ -104,7 +104,7 @@ private static boolean checkTableHeader(Page header) { * @param fileName 文件名 * @return 解析或新建得到的数据项管理器对象 */ - public static IItemStorage ofFile(String fileName) throws FileException, PageException, IOException, LogException { + public static IItemStorage ofFile(String fileName) throws FileException, PageException, LogException { var pageStorage = PageManager.fromFile(fileName); var header = pageStorage.get(0); header.pin(); @@ -137,7 +137,7 @@ public static IItemStorage ofFile(String fileName) throws FileException, PageExc /** * 不使用日志打开文件 */ - public static IItemStorage ofFileWithoutLog(String fileName) throws FileException, PageException, IOException, LogException { + public static IItemStorage ofFileWithoutLog(String fileName) throws FileException, PageException, PageCorruptedException { var pageStorage = PageManager.fromFile(fileName); var header = pageStorage.get(0); header.pin(); @@ -172,7 +172,7 @@ public static IItemStorage ofFileWithoutLog(String fileName) throws FileExceptio * @param metadata 表头信息 * @return 数据项管理器 */ - public static IItemStorage ofNewFile(TransactionContext txContext, String fileName, byte[] metadata) throws IOException, FileException, PageException, LogException { + public static IItemStorage ofNewFile(TransactionContext txContext, String fileName, byte[] metadata) throws FileException, PageException, LogException { var pageStorage = ItemStorage.ofFile(fileName); pageStorage.setMetadata(txContext, metadata); return pageStorage; @@ -181,7 +181,7 @@ public static IItemStorage ofNewFile(TransactionContext txContext, String fileNa /** * 不使用日志的创建文件 */ - public static IItemStorage ofNewFileWithoutLog(String fileName, byte[] metadata) throws IOException, LogException, FileException, PageException { + public static IItemStorage ofNewFileWithoutLog(String fileName, byte[] metadata) throws FileException, PageException { var pageStorage = ItemStorage.ofFileWithoutLog(fileName); pageStorage.setMetadataWithoutLog(metadata); return pageStorage; @@ -264,7 +264,7 @@ private static byte[] parseData(Page page, Item item) throws PageCorruptedExcept * @param page 页 * @return 该页是否已经被初始化 */ - private boolean checkPageHeader(Page page) { + private boolean checkPageHeader(Page page) throws PageCorruptedException{ byte[] pageFlag = new byte[4]; try { var n = page.getData().read(pageFlag); @@ -291,7 +291,7 @@ private Optional getPageHeader(Page page) { return Optional.empty(); } - private PageHeader initPage(Page page) { + private PageHeader initPage(Page page) throws PageCorruptedException { final byte[] bytes; try { bytes = JBBPOut.BeginBin(). @@ -310,7 +310,7 @@ private PageHeader initPage(Page page) { /** * 修改当前第一个可用页 */ - private void addTempFreePage() { + private void addTempFreePage() throws PageCorruptedException { this.tempFreePage += 1; var page = pageStorage.get(0); page.pin(); @@ -328,7 +328,7 @@ private void addTempFreePage() { } @Override - public void setMetaUuid(long uuid) throws IOException, PageException { + public void setMetaUuid(long uuid) throws PageException, IOException { var page = getPage(0); try { var bytes = JBBPOut.BeginBin(). @@ -365,13 +365,13 @@ public void flush(long uuid) throws FileException { } @Override - public synchronized long insertItem(TransactionContext txContext, byte[] item) throws IOException, FileException { + public synchronized long insertItem(TransactionContext txContext, byte[] item) throws LogException,PageCorruptedException { long uuid = insertItemWithoutLog(item); recoveryStorage.insert(txContext.getXid(),uuid,item); return uuid; } @Override - public synchronized long insertItemWithoutLog(byte[] item) { + public synchronized long insertItemWithoutLog(byte[] item) throws PageCorruptedException{ var page = getPage(this.tempFreePage); try { var pageHeaderOp = getPageHeader(page); @@ -470,12 +470,9 @@ public synchronized void insertItemWithUuid(byte[] item, long uuid) { // 若存在则不需要恢复，直接返回 } - /** - * 日志恢复时用，回退对应的insert操作 - * @param uuid - */ + @Override - public void deleteUuid(long uuid) throws IOException, PageException { + public void deleteUuid(long uuid) throws PageException, IOException { var page = getPage(uuid); try { var rnd = getRndByUuid(uuid); @@ -586,7 +583,7 @@ public List listItemByPageId(int pageId) { } @Override - public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, FileException, IOException { + public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, LogException { var item_before = updateItemWithoutLog(uuid, item); recoveryStorage.update(txContext.getXid(),uuid,item_before,item); } @@ -650,7 +647,7 @@ public byte[] getMetadata() { } @Override - public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, IOException, FileException { + public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, LogException { var page = getPage(0); try{ var header = parseTableHeader(page); diff --git a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java index 5d2585b..d692842 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/IRecoveryStorage.java @@ -19,21 +19,21 @@ public interface IRecoveryStorage { * @param xid 事务编号 * @param snapshots 快照集合 */ - void begin(int xid, List snapshots) throws IOException, FileException; + void begin(int xid, List snapshots) throws LogException; /** * 记录事务失败回滚 * * @param xid */ - void rollback(int xid) throws IOException, FileException; + void rollback(int xid) throws LogException; /** * 记录事务完成 * * @param xid */ - void commit(int xid) throws IOException, FileException; + void commit(int xid) throws LogException; /** * 插入数据项的日志记录 @@ -42,7 +42,7 @@ public interface IRecoveryStorage { * @param uuid 数据项的对应编号 * @param item 插入的数据内容 */ - void insert(int xid, long uuid, byte[] item) throws IOException, FileException; + void insert(int xid, long uuid, byte[] item) throws LogException; /** * 更新数据项的日志记录 @@ -50,14 +50,14 @@ public interface IRecoveryStorage { * @param xid * @param uuid */ - void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws IOException, FileException; + void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws LogException; /** * 更新数据项的日志头 * * @param xid */ - void updateMeta(int xid, long beforeUuid,byte[] metadata) throws IOException, FileException; + void updateMeta(int xid, long beforeUuid,byte[] metadata) throws LogException; /** * 模拟打印日志资料 @@ -67,5 +67,5 @@ public interface IRecoveryStorage { /** * 恢复数据 */ - void recovery() throws IOException, LogException, FileException, PageException; + void recovery() throws LogException; } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index 6fb4be7..ae3b0b0 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -19,16 +19,16 @@ public class RecoveryManager { * @param fileName 文件名 * @return 数据库日志管理器 */ - public static void recovery(String fileName) throws PageException, LogException, FileException, IOException { + public static void recovery(String fileName) throws LogException { RecoveryStorage.ofFile(fileName).recovery(); } - public static IRecoveryStorage getRecoveryStorage(String fileName) throws FileException, IOException, LogException, PageException { + public static IRecoveryStorage getRecoveryStorage(String fileName) throws LogException { return RecoveryStorage.ofFile(fileName); } - public static IRecoveryStorage createRecoveryStorage(String fileName) throws IOException, FileException, PageException, LogException { + public static IRecoveryStorage createRecoveryStorage(String fileName) throws LogException { return RecoveryStorage.ofNewFile(fileName); } } diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index 80b7e04..d0c0f91 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -6,6 +6,7 @@ import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.ItemManager; import net.kaaass.rumbase.dataitem.exception.UUIDException; +import net.kaaass.rumbase.page.Page; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; @@ -43,9 +44,13 @@ public RecoveryStorage(IItemStorage itemStorage,String fileName) { * @param fileName * @return 日志管理器 */ - public static IRecoveryStorage ofFile(String fileName) throws FileException, IOException, LogException, PageException { - var itemStorage = ItemManager.fromFileWithoutLog(fileName + ".log"); - return new RecoveryStorage(itemStorage,fileName); + public static IRecoveryStorage ofFile(String fileName) throws LogException { + try { + var itemStorage = ItemManager.fromFileWithoutLog(fileName + ".log"); + return new RecoveryStorage(itemStorage,fileName); + }catch (FileException | PageException e){ + throw new LogException(10); + } } /** @@ -53,85 +58,114 @@ public static IRecoveryStorage ofFile(String fileName) throws FileException, IOE * @param fileName 文件名 * @return */ - public static IRecoveryStorage ofNewFile(String fileName) throws FileException, IOException, PageException, LogException { - var metadata = JBBPOut.BeginBin().Int(HEADER).End().toByteArray(); - var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log",metadata); - return new RecoveryStorage(itemStorage,fileName); + public static IRecoveryStorage ofNewFile(String fileName) throws LogException { + try { + var metadata = JBBPOut.BeginBin().Int(HEADER).End().toByteArray(); + var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log",metadata); + return new RecoveryStorage(itemStorage,fileName); + }catch (IOException | FileException | PageException e) { + throw new LogException(9); + } } @Override - public void begin(int xid, List snapshots) throws IOException, FileException { - var jbbp = JBBPOut.BeginBin(). - Byte(TX_BEGIN). - Int(xid). - Int(snapshots.size()); - for ( var i : snapshots){ - jbbp = jbbp.Int(i); + public void begin(int xid, List snapshots) throws LogException { + try { + var jbbp = JBBPOut.BeginBin(). + Byte(TX_BEGIN). + Int(xid). + Int(snapshots.size()); + for ( var i : snapshots){ + jbbp = jbbp.Int(i); + } + var bytes = jbbp.End().toByteArray(); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); + } catch (IOException | FileException e) { + throw new LogException(3); } - var bytes = jbbp.End().toByteArray(); - var uuid = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(uuid); } @Override - public void rollback(int xid) throws IOException, FileException { - var bytes = JBBPOut.BeginBin(). - Byte(TX_ABORT). - Int(xid). - End().toByteArray(); - var uuid = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(uuid); + public void rollback(int xid) throws LogException { + try { + var bytes = JBBPOut.BeginBin(). + Byte(TX_ABORT). + Int(xid). + End().toByteArray(); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); + }catch (FileException | IOException e){ + throw new LogException(4); + } } @Override - public void commit(int xid) throws IOException, FileException { - var bytes = JBBPOut.BeginBin(). - Byte(TX_COMMIT). - Int(xid). - End().toByteArray(); - var uuid = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(uuid); + public void commit(int xid) throws LogException { + try { + var bytes = JBBPOut.BeginBin(). + Byte(TX_COMMIT). + Int(xid). + End().toByteArray(); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); + }catch (FileException | IOException e){ + throw new LogException(5); + } + } @Override - public void insert(int xid, long uuid, byte[] item) throws IOException, FileException { - var bytes = JBBPOut.BeginBin(). - Byte(INSERT_FLAG). - Int(xid). - Long(uuid). - Int(item.length). - Byte(item). - End().toByteArray(); - var id = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(id); + public void insert(int xid, long uuid, byte[] item) throws LogException { + try { + var bytes = JBBPOut.BeginBin(). + Byte(INSERT_FLAG). + Int(xid). + Long(uuid). + Int(item.length). + Byte(item). + End().toByteArray(); + var id = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(id); + }catch (FileException | IOException e){ + throw new LogException(6); + } } @Override - public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws IOException, FileException { - var bytes = JBBPOut.BeginBin(). - Byte(UPDATE_FLAG). - Int(xid). - Long(uuid). - Int(itemBefore.length). - Byte(itemBefore). - Int(itemAfter.length). - Byte(itemAfter). - End().toByteArray(); - var id = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(id); + public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws LogException { + try{ + var bytes = JBBPOut.BeginBin(). + Byte(UPDATE_FLAG). + Int(xid). + Long(uuid). + Int(itemBefore.length). + Byte(itemBefore). + Int(itemAfter.length). + Byte(itemAfter). + End().toByteArray(); + var id = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(id); + }catch (FileException | IOException e){ + throw new LogException(7); + } } @Override - public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws IOException, FileException { - var bytes =JBBPOut.BeginBin(). - Byte(METADATA_UPDATE_FLAG). - Int(xid). - Long(beforeUuid). - Int(metadata.length). - Byte(metadata). - End().toByteArray(); - var uuid = logStorage.insertItemWithoutLog(bytes); - logStorage.flush(uuid); + public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws LogException { + try { + var bytes =JBBPOut.BeginBin(). + Byte(METADATA_UPDATE_FLAG). + Int(xid). + Long(beforeUuid). + Int(metadata.length). + Byte(metadata). + End().toByteArray(); + var uuid = logStorage.insertItemWithoutLog(bytes); + logStorage.flush(uuid); + }catch (FileException | IOException e){ + throw new LogException(8); + } } @@ -151,24 +185,33 @@ public List getContent() { } @Override - public void recovery() throws IOException, LogException, FileException, PageException { - var itemStorage = ItemManager.fromFile(this.fileName); - var logs = getContent(); - var xidMaps = parseXid(logs); - for (var log : logs){ - parseLog(log,itemStorage,xidMaps); + public void recovery() throws LogException{ + try { + var itemStorage = ItemManager.fromFile(this.fileName); + var logs = getContent(); + var xidMaps = parseXid(logs); + for (var log : logs){ + parseLog(log,itemStorage,xidMaps); + } + }catch (FileException | PageException e){ + throw new LogException(11); } } /** * 先进行一遍解析，得到所有已完成的事务和未完成的事务，来决定redo还是undo */ - private Map> parseXid(List logs) throws IOException { + private Map> parseXid(List logs) throws LogException { List commitXids = new ArrayList<>(); List abortXids = new ArrayList<>(); for (var log : logs){ - var tx = parseTx(log); + Tx tx = null; + try { + tx = parseTx(log); + } catch (IOException e) { + throw new LogException(1); + } switch (tx.type){ case TX_COMMIT: // 若是commit 则放入commitXid并将对应的xid移出abort @@ -242,52 +285,57 @@ private boolean checkCommit(int xid,Map> maps) throws Lo throw new LogException(2); } } - private void parseLog(byte[] log, IItemStorage itemStorage, Map> xidMaps) throws IOException, LogException, PageException { - var type = parseType(log); - switch (type){ - case INSERT_FLAG: - System.out.println("正在解析插入"); - var insertLog = parseInsert(log); - if (checkCommit(insertLog.xid,xidMaps)){ - // 如果事务已经提交，则redo - itemStorage.insertItemWithUuid(insertLog.item,insertLog.uuid); - }else { - // 如果事务没有提交，则undo - itemStorage.deleteUuid(insertLog.uuid); - } - break; - case UPDATE_FLAG: - System.out.println("正在解析更新"); - var updateLog = parseUpdate(log); - if (checkCommit(updateLog.xid,xidMaps)){ - // 若事务已经提交则redo - try { - itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemAfter); - } catch (UUIDException ignored) { - // uuid不存在说明对应之前的事务没有执行，是正常 - } - }else { - // 若事务没有提交，则undo,恢复之前的数据 - try { - itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemBefore); - } catch (UUIDException ignored) { + private void parseLog(byte[] log, IItemStorage itemStorage, Map> xidMaps) throws LogException { + try { + var type = parseType(log); + switch (type){ + case INSERT_FLAG: + System.out.println("正在解析插入"); + var insertLog = parseInsert(log); + if (checkCommit(insertLog.xid,xidMaps)){ + // 如果事务已经提交，则redo + itemStorage.insertItemWithUuid(insertLog.item,insertLog.uuid); + }else { + // 如果事务没有提交，则undo + itemStorage.deleteUuid(insertLog.uuid); } - } - break; - case METADATA_UPDATE_FLAG: - System.out.println("正在解析头信息更新"); - var updateMetaLog = parseUpdateMeta(log); - if (checkCommit(updateMetaLog.xid,xidMaps)){ - // redo - itemStorage.setMetadataWithoutLog(updateMetaLog.metadata); - }else { - // undo - itemStorage.setMetaUuid(updateMetaLog.beforeUuid); - } - break; - default: - return; + break; + case UPDATE_FLAG: + System.out.println("正在解析更新"); + var updateLog = parseUpdate(log); + if (checkCommit(updateLog.xid,xidMaps)){ + // 若事务已经提交则redo + try { + itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemAfter); + } catch (UUIDException ignored) { + // uuid不存在说明对应之前的事务没有执行，是正常 + } + }else { + // 若事务没有提交，则undo,恢复之前的数据 + try { + itemStorage.updateItemWithoutLog(updateLog.uuid,updateLog.itemBefore); + } catch (UUIDException ignored) { + + } + } + break; + case METADATA_UPDATE_FLAG: + System.out.println("正在解析头信息更新"); + var updateMetaLog = parseUpdateMeta(log); + if (checkCommit(updateMetaLog.xid,xidMaps)){ + // redo + itemStorage.setMetadataWithoutLog(updateMetaLog.metadata); + }else { + // undo + itemStorage.setMetaUuid(updateMetaLog.beforeUuid); + } + break; + default: + return; + } + }catch (IOException | PageException e){ + throw new LogException(1); } } diff --git a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java index e22f074..6b6ae8a 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java +++ b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java @@ -13,10 +13,19 @@ public class LogException extends RumbaseException { public static final Map REASONS = new HashMap() {{ put(1, "日志文件无法解析"); - put(2,"事务ID不存在"); + put(2, "事务ID不存在"); + put(3, "事务提交日志写回错误"); + put(4, "事务回滚日志写回错误"); + put(5, "事务提交日志写回错误"); + put(6, "插入数据项日志写回错误"); + put(7, "更新数据项日志写回错误"); + put(8, "更新头信息日志写回错误"); + put(9, "日志文件创建失败"); + put(10, "日志文件打开失败"); + put(11, "日志获取原文件失败"); }}; public LogException(int subId) { - super(9001,subId,REASONS.get(subId)); + super(3001,subId,REASONS.get(subId)); } } From 88f3055ea0ff9a73b885ead3ace96926fb6985e5 Mon Sep 17 00:00:00 2001 From: KAAAsS Date: Sun, 17 Jan 2021 13:58:22 +0800 Subject: [PATCH 19/20] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E4=BB=A5=E9=80=9A?= =?UTF-8?q?=E8=BF=87=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/IItemStorage.java | 25 +++- .../kaaass/rumbase/dataitem/ItemStorage.java | 32 ++++- .../dataitem/mock/MockItemStorage.java | 4 + .../kaaass/rumbase/page/RumPageStorage.java | 25 ++-- .../rumbase/record/MvccRecordStorage.java | 19 +-- .../recovery/mock/MockRecoveryStorage.java | 5 +- .../rumbase/dataitem/IItemStorageTest.java | 83 ++++++------ .../rumbase/recovery/IRecoveryTest.java | 128 +++++++++--------- testInsert.db | Bin 40960 -> 0 bytes 9 files changed, 174 insertions(+), 147 deletions(-) delete mode 100644 testInsert.db diff --git a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java index fb94da3..bc0c2e3 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/IItemStorage.java @@ -22,31 +22,35 @@ public interface IItemStorage { /** * 获得日志管理器 + * * @return */ IRecoveryStorage getRecoveryStorage(); /** * 获取表的tempFreePage + * * @return */ public int getMaxPageId(); /** - * 将uuid对应的页强制写回 + * 将uuid对应的页强制写回 */ public void flush(long uuid) throws FileException; + /** * 插入数据项 * * @param txContext 事务上下文 - * @param item 数据项 + * @param item 数据项 * @return 返回数据项的UUID */ - long insertItem(TransactionContext txContext, byte[] item) throws LogException,PageCorruptedException; + long insertItem(TransactionContext txContext, byte[] item) throws PageCorruptedException; /** * 不用日志进行插入，用于日志的管理 + * * @param item 数据项 * @return uuid */ @@ -89,9 +93,9 @@ public interface IItemStorage { * @param item 数据项 * @throws UUIDException 没有找到对应UUID的异常 */ - void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, LogException,PageCorruptedException; + void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException; - byte[] updateItemWithoutLog(long uuid,byte[] item) throws UUIDException; + byte[] updateItemWithoutLog(long uuid, byte[] item) throws UUIDException; /** * 获得数据项存储的元数据（可以用于头） @@ -105,10 +109,11 @@ public interface IItemStorage { * * @param metadata 头信息 */ - long setMetadata(TransactionContext txContext, byte[] metadata) throws LogException; + long setMetadata(TransactionContext txContext, byte[] metadata); /** - * 不使用日志设置元数据 + * 不使用日志设置元数据 + * * @param metadata 头信息 */ long setMetadataWithoutLog(byte[] metadata); @@ -122,9 +127,15 @@ public interface IItemStorage { /** * 日志恢复时用，回退对应的insert操作，做法是删除对应的uuid + * * @param uuid */ void deleteUuid(long uuid) throws PageException, IOException; + + /** + * 建议存储进行一次回写 + */ + void flush(); } diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java index a21e5c8..016c8d1 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemStorage.java @@ -368,9 +368,14 @@ public void flush(long uuid) throws FileException { } @Override - public synchronized long insertItem(TransactionContext txContext, byte[] item) throws LogException,PageCorruptedException { + public synchronized long insertItem(TransactionContext txContext, byte[] item) throws PageCorruptedException { long uuid = insertItemWithoutLog(item); - recoveryStorage.insert(txContext.getXid(),uuid,item); + try { + recoveryStorage.insert(txContext.getXid(),uuid,item); + } catch (LogException e) { + log.warn("文件 {} 日志写入出现故障，触发回写", this.fileName, e); + flush(); + } return uuid; } @Override @@ -497,6 +502,11 @@ public void deleteUuid(long uuid) throws PageException, IOException { } } + @Override + public void flush() { + pageStorage.flush(); + } + /** * 检查uuid是否存在,若Uuid的页号超过当前可用页，则直接返回False * @@ -586,9 +596,14 @@ public List listItemByPageId(int pageId) { } @Override - public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException, LogException { + public void updateItemByUuid(TransactionContext txContext, long uuid, byte[] item) throws UUIDException, PageCorruptedException { var item_before = updateItemWithoutLog(uuid, item); - recoveryStorage.update(txContext.getXid(),uuid,item_before,item); + try { + recoveryStorage.update(txContext.getXid(),uuid,item_before,item); + } catch (LogException e) { + log.warn("文件 {} 日志写入出现故障，触发回写", this.fileName, e); + flush(); + } } @Override @@ -650,12 +665,17 @@ public byte[] getMetadata() { } @Override - public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException, LogException { + public long setMetadata(TransactionContext txContext, byte[] metadata) throws PageCorruptedException { var page = getPage(0); try{ var header = parseTableHeader(page); var uuid = setMetadataWithoutLog(metadata); - recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); + try { + recoveryStorage.updateMeta(txContext.getXid(),header.headerUuid,metadata); + } catch (LogException e) { + log.warn("文件 {} 日志写入出现故障，触发回写", this.fileName, e); + flush(); + } return uuid; }finally { releasePage(page); diff --git a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java index 5010d7d..e2fdad7 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/mock/MockItemStorage.java @@ -177,6 +177,10 @@ public void deleteUuid(long uuid) throws IOException, PageException { } + @Override + public void flush() { + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java b/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java index d94dc23..1f20f82 100644 --- a/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java +++ b/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java @@ -1,21 +1,22 @@ package net.kaaass.rumbase.page; +import lombok.extern.slf4j.Slf4j; import net.kaaass.rumbase.page.exception.BufferException; import net.kaaass.rumbase.page.exception.FileException; import java.io.*; import java.util.Arrays; -import java.util.HashMap; import java.util.Map; -import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; /** * @author 11158 */ +@Slf4j public class RumPageStorage implements PageStorage { - public RumPageStorage(String filepath) throws FileException { + public RumPageStorage(String filepath) { this.filepath = filepath; - pageMap = new HashMap<>(); + pageMap = new ConcurrentHashMap<>(); } @Override @@ -40,7 +41,7 @@ public Page get(long pageId) { while (in.available() < (pageId + 1 + PageManager.FILE_HEAD_SIZE) * PageManager.PAGE_SIZE) { FileWriter fw = new FileWriter(file, true); char[] blank = new char[PageManager.PAGE_SIZE * (in.available() / PageManager.PAGE_SIZE)]; - Arrays.fill(blank, (char)0); + Arrays.fill(blank, (char) 0); fw.write(blank); fw.close(); } @@ -59,7 +60,8 @@ public Page get(long pageId) { } int offset = -1; while (offset < 0) { - synchronized (RumBuffer.getInstance()) {//并非区间锁，而是将整个内存全部锁住 + //并非区间锁，而是将整个内存全部锁住 + synchronized (RumBuffer.getInstance()) { try { offset = RumBuffer.getInstance().getFreeOffset(); RumBuffer.getInstance().put(offset, data); @@ -86,12 +88,15 @@ public Page get(long pageId) { @Override public void flush() { - Set> entrySet = this.pageMap.entrySet(); - for (Map.Entry entry : entrySet) { + var entrySet = this.pageMap.entrySet(); + for (var entry : entrySet) { + var page = entry.getValue(); try { - entry.getValue().flush(); + if (page instanceof RumPage && ((RumPage) page).dirty()) { + page.flush(); + } } catch (Exception e) { - e.printStackTrace(); + log.warn("回写页面 {} 发生异常", entry.getKey()); } } } diff --git a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java index d33ac44..de87af2 100644 --- a/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java +++ b/src/main/java/net/kaaass/rumbase/record/MvccRecordStorage.java @@ -11,6 +11,7 @@ import net.kaaass.rumbase.record.exception.NeedRollbackException; import net.kaaass.rumbase.record.exception.RecordNotFoundException; import net.kaaass.rumbase.record.exception.StorageCorruptedException; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import net.kaaass.rumbase.transaction.TransactionIsolation; import net.kaaass.rumbase.transaction.TransactionStatus; @@ -49,13 +50,7 @@ public long insert(TransactionContext txContext, byte[] rawData) { writePayload(data, rawData); // 不用检查版本跳跃的原因是，插入本身不用；更新操作必定先删除，而删除检查 // 插入记录 - try { - return storage.insertItem(txContext, data); - } catch (IOException | FileException e) { - e.printStackTrace(); - // FIXME - return -1; - } + return storage.insertItem(txContext, data); } @Override @@ -122,9 +117,6 @@ public void delete(TransactionContext txContext, long recordId) throws RecordNot storage.updateItemByUuid(txContext, recordId, data); } catch (UUIDException e) { throw new RecordNotFoundException(1, e); - } catch (IOException | FileException e) { - e.printStackTrace(); - // FIXME } } @@ -244,12 +236,7 @@ public void setMetadata(TransactionContext txContext, byte[] metadata) { var newMetadata = new byte[oldMetadata.length + 8]; System.arraycopy(oldMetadata, 0, newMetadata, 8, oldMetadata.length); MvccUtil.writeLong(newMetadata, 0, newId); - try { - storage.setMetadata(txContext, newMetadata); - } catch (IOException | FileException e) { - e.printStackTrace(); - // FIXME - } + storage.setMetadata(txContext, newMetadata); // 缓存取消 this.metadataCache = null; } diff --git a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java index b872dbe..b22049d 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/mock/MockRecoveryStorage.java @@ -69,9 +69,6 @@ public List getContent() { } @Override - public void recovery() throws IOException { - + public void recovery() { } - - } diff --git a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java index 3e8ccfa..c0fad1f 100644 --- a/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java +++ b/src/test/java/net/kaaass/rumbase/dataitem/IItemStorageTest.java @@ -1,7 +1,7 @@ package net.kaaass.rumbase.dataitem; -import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; +import net.kaaass.rumbase.FileUtil; import net.kaaass.rumbase.dataitem.exception.PageCorruptedException; import net.kaaass.rumbase.dataitem.exception.UUIDException; import net.kaaass.rumbase.page.exception.FileException; @@ -9,7 +9,6 @@ import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; import org.junit.AfterClass; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -32,6 +31,8 @@ @Slf4j public class IItemStorageTest { + private static final String PATH = FileUtil.TEST_PATH; + @BeforeClass public static void createDataFolder() { FileUtil.prepare(); @@ -42,19 +43,17 @@ public static void clearDataFolder() { FileUtil.clear(); } - private static final String PATH = FileUtil.TEST_PATH; - /** * 测试能否从已有文件中解析得到数据项管理器 */ @Test - public void testGetFromFile() throws FileException, IOException, PageException, LogException { + public void testGetFromFile() throws FileException, PageException, LogException { String fileName = PATH + "testGetFromFile.db"; - var itemStorage = ItemManager.fromFile(fileName); + ItemManager.fromFile(fileName); // 如果表中没有对应的文件，那么就抛出错误 String failFileName = "error.db"; try { - IItemStorage iItemStorage1 = ItemManager.fromFile(failFileName); + ItemManager.fromFile(failFileName); } catch (FileException f) { log.error("Exception Error :", f); } @@ -64,15 +63,15 @@ public void testGetFromFile() throws FileException, IOException, PageException, * 测试能否新建文件并得到数据项管理器 */ @Test - public void testCreateFile() throws IOException, FileException, PageException, LogException { + public void testCreateFile() throws FileException, PageException, LogException { TransactionContext txContext = TransactionContext.empty(); String fileName = PATH + "testCreateFile.db"; byte[] metadata = new byte[1024]; // 第一次执行的时候，表中没有数据，不会报错 - var iItemStorage = ItemManager.createFile(txContext, fileName, metadata); + ItemManager.createFile(txContext, fileName, metadata); try { - iItemStorage = ItemManager.createFile(txContext, fileName, metadata); + ItemManager.createFile(txContext, fileName, metadata); fail("should get exception"); } catch (Exception e) { log.error("Exception Error :", e); @@ -82,7 +81,7 @@ public void testCreateFile() throws IOException, FileException, PageException, L /** * 进行插入的测试 */ - public void testInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = PATH + "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -99,7 +98,7 @@ public void testInsert() throws FileException, IOException, PageException, UUIDE /** * 对插入一个已分配UUID的测试 */ - public void testInsertWithUUID() throws FileException, IOException, PageException { + public void testInsertWithUUID() throws FileException, IOException, PageException, LogException { String fileName = PATH + "testInsertWithUUID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -123,7 +122,7 @@ public void testInsertWithUUID() throws FileException, IOException, PageExceptio /** * 对插入大量数据进行测试 */ - public void testManyInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testManyInsert() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = PATH + "testInsertMany.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -143,7 +142,7 @@ public void testManyInsert() throws FileException, IOException, PageException, U /** * 获取整个页的数据项进行测试 */ - public void testQueryByPageID() throws FileException, IOException, PageException, PageCorruptedException { + public void testQueryByPageID() throws FileException, IOException, PageException, PageCorruptedException, LogException { String fileName = PATH + "testQueryByPageID.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -178,35 +177,10 @@ public void testQueryByPageID() throws FileException, IOException, PageException } } - - static class Insert implements Runnable { - IItemStorage iItemStorage; - TransactionContext txContext; - - public Insert(IItemStorage iItemStorage, TransactionContext txContext) { - this.iItemStorage = iItemStorage; - this.txContext = txContext; - } - - @Override - public void run() { - var bytes = new byte[]{1, 2, 3, 4}; - try { - for (int i = 0; i < 100; i++) { - long uuid = iItemStorage.insertItem(txContext, bytes); - assertArrayEquals(bytes, iItemStorage.queryItemByUuid(uuid)); - } - } catch (Exception e) { - e.printStackTrace(); - fail("Exception caught"); - } - } - } - /** * 测试并发下插入是否有问题 */ - public void testSynInsert() throws IOException, FileException, PageException { + public void testSynInsert() throws IOException, FileException, PageException, LogException { String fileName = PATH + "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; @@ -217,11 +191,10 @@ public void testSynInsert() throws IOException, FileException, PageException { new Thread(new Insert(iItemStorage, txContext)).start(); } - /** * 对更新进行测试 */ - public void testUpdate() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testUpdate() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = PATH + "testUpdate.db"; TransactionContext txContext = TransactionContext.empty(); IItemStorage iItemStorage = ItemManager.fromFile(fileName); @@ -248,7 +221,7 @@ public void testUpdate() throws FileException, IOException, PageException, UUIDE /** * 测试修改和获取表头信息 */ - public void testMeta() throws FileException, IOException, PageException, UUIDException, PageCorruptedException { + public void testMeta() throws FileException, IOException, PageException, UUIDException, PageCorruptedException, LogException { String fileName = PATH + "testMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] result = new byte[]{1, 2, 3, 4}; @@ -258,4 +231,28 @@ public void testMeta() throws FileException, IOException, PageException, UUIDExc assertArrayEquals(result, bs); } + static class Insert implements Runnable { + IItemStorage iItemStorage; + TransactionContext txContext; + + public Insert(IItemStorage iItemStorage, TransactionContext txContext) { + this.iItemStorage = iItemStorage; + this.txContext = txContext; + } + + @Override + public void run() { + var bytes = new byte[]{1, 2, 3, 4}; + try { + for (int i = 0; i < 100; i++) { + long uuid = iItemStorage.insertItem(txContext, bytes); + assertArrayEquals(bytes, iItemStorage.queryItemByUuid(uuid)); + } + } catch (Exception e) { + e.printStackTrace(); + fail("Exception caught"); + } + } + } + } diff --git a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java index 3197f88..9de183a 100644 --- a/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java +++ b/src/test/java/net/kaaass/rumbase/recovery/IRecoveryTest.java @@ -2,6 +2,7 @@ import junit.framework.TestCase; import lombok.extern.slf4j.Slf4j; +import net.kaaass.rumbase.FileUtil; import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.ItemManager; import net.kaaass.rumbase.dataitem.exception.UUIDException; @@ -9,26 +10,38 @@ import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Random; -import static org.junit.Assert.assertArrayEquals; - /** * 对日志进行保存和恢复 */ @Slf4j -public class IRecoveryTest extends TestCase { +public class IRecoveryTest { + + @BeforeClass + public static void createDataFolder() { + FileUtil.prepare(); + } + + @AfterClass + public static void clearDataFolder() { + FileUtil.clear(); + } + public static final String PATH = FileUtil.TEST_PATH; + + @Test public void testInsert() throws PageException, LogException, FileException, IOException, UUIDException { - new File("test_gen_files/testInsert.db").delete(); - new File("test_gen_files/testInsert.db.log").delete(); - String fileName = "test_gen_files/testInsert.db"; + String fileName = PATH + "testInsert.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; var txContext = TransactionContext.empty(); @@ -37,58 +50,56 @@ public void testInsert() throws PageException, LogException, FileException, IOEx var xid2 = 0; List snaps = new ArrayList<>(); // 测试日志中存在，但是表中不存在进行恢复 - recoveryStorage.begin(xid,snaps); + recoveryStorage.begin(xid, snaps); long a = 1; long uuid = (a << 32) + Math.abs(new Random().nextInt()); - recoveryStorage.insert(xid,uuid,bytes); + recoveryStorage.insert(xid, uuid, bytes); recoveryStorage.commit(xid); // 测试日志中存在，且数据中也存在，会不会重复插入 - recoveryStorage.begin(xid2,snaps); - iItemStorage.insertItem(txContext,bytes); + recoveryStorage.begin(xid2, snaps); + iItemStorage.insertItem(txContext, bytes); recoveryStorage.commit(xid2); recoveryStorage.recovery(); // 测试日志有额外的数据有没有被插入 var result = iItemStorage.queryItemByUuid(uuid); - assertTrue(Arrays.equals(result,bytes)); + Assert.assertTrue(Arrays.equals(result, bytes)); // 测试表中有的数据是否被重复插入 var list = iItemStorage.listItemByPageId(1); - assertEquals(2,list.size()); + Assert.assertEquals(2, list.size()); } + @Test public void testInsertFail() throws PageException, LogException, FileException, IOException, UUIDException { - new File("test_gen_files/testInsertFailed.db").delete(); - new File("test_gen_files/testInsertFailed.db.log").delete(); - String fileName = "test_gen_files/testInsertFailed.db"; + String fileName = PATH + "testInsertFailed.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; var txContext = TransactionContext.empty(); var recoveryStorage = iItemStorage.getRecoveryStorage(); var xid = 0; List snaps = new ArrayList<>(); - recoveryStorage.begin(xid,snaps); - long uuid = iItemStorage.insertItem(txContext,bytes); + recoveryStorage.begin(xid, snaps); + long uuid = iItemStorage.insertItem(txContext, bytes); recoveryStorage.rollback(xid); recoveryStorage.recovery(); try { var item = iItemStorage.queryItemByUuid(uuid); - assertFalse(Arrays.equals(bytes,item)); - }catch (Exception e){ + Assert.assertFalse(Arrays.equals(bytes, item)); + } catch (Exception e) { } } + @Test public void testUpdate() throws PageException, LogException, FileException, IOException, UUIDException { - new File("test_gen_files/testUpdate.db").delete(); - new File("test_gen_files/testUpdate.db.log").delete(); - String fileName = "test_gen_files/testUpdate.db"; + String fileName = PATH + "testUpdate.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - byte[] bytesUpdate = new byte[]{2,3,4,5}; + byte[] bytesUpdate = new byte[]{2, 3, 4, 5}; var txContext = TransactionContext.empty(); var recoveryStorage = iItemStorage.getRecoveryStorage(); @@ -96,96 +107,91 @@ public void testUpdate() throws PageException, LogException, FileException, IOEx List snaps = new ArrayList<>(); // 测试在表中存在的redo - recoveryStorage.begin(xid,snaps); - long uuid = iItemStorage.insertItem(txContext,bytes); - iItemStorage.updateItemByUuid(txContext,uuid,bytesUpdate); + recoveryStorage.begin(xid, snaps); + long uuid = iItemStorage.insertItem(txContext, bytes); + iItemStorage.updateItemByUuid(txContext, uuid, bytesUpdate); recoveryStorage.commit(xid); // 测试在表中不存在的redo int xid2 = 1; - recoveryStorage.begin(xid2,snaps); + recoveryStorage.begin(xid2, snaps); long a = 1; long uuid2 = (a << 32) + Math.abs(new Random().nextInt()); - recoveryStorage.insert(xid2,uuid2,bytes); - recoveryStorage.update(xid2,uuid2,bytes,bytesUpdate); + recoveryStorage.insert(xid2, uuid2, bytes); + recoveryStorage.update(xid2, uuid2, bytes, bytesUpdate); recoveryStorage.commit(xid2); // 测试abort的事务 int xid3 = 2; - recoveryStorage.begin(xid3,snaps); + recoveryStorage.begin(xid3, snaps); long b = 1; long uuid3 = (b << 32) + Math.abs(new Random().nextInt()); - recoveryStorage.insert(xid3,uuid3,bytes); + recoveryStorage.insert(xid3, uuid3, bytes); recoveryStorage.commit(xid3); int xid4 = 3; - recoveryStorage.begin(xid4,snaps); - recoveryStorage.update(xid4,uuid3,bytes,bytesUpdate); + recoveryStorage.begin(xid4, snaps); + recoveryStorage.update(xid4, uuid3, bytes, bytesUpdate); recoveryStorage.rollback(xid4); recoveryStorage.recovery(); - assertTrue(Arrays.equals(bytesUpdate,iItemStorage.queryItemByUuid(uuid))); - assertTrue(Arrays.equals(bytesUpdate,iItemStorage.queryItemByUuid(uuid2))); - assertTrue(Arrays.equals(bytes,iItemStorage.queryItemByUuid(uuid3))); + Assert.assertTrue(Arrays.equals(bytesUpdate, iItemStorage.queryItemByUuid(uuid))); + Assert.assertTrue(Arrays.equals(bytesUpdate, iItemStorage.queryItemByUuid(uuid2))); + Assert.assertTrue(Arrays.equals(bytes, iItemStorage.queryItemByUuid(uuid3))); var list = iItemStorage.listItemByPageId(1); - assertEquals(3,list.size()); + Assert.assertEquals(3, list.size()); } + @Test public void testUpdateMeta() throws PageException, LogException, FileException, IOException { - new File("test_gen_files/testUpdateMeta.db").delete(); - new File("test_gen_files/testUpdateMeta.db.log").delete(); - String fileName = "test_gen_files/testUpdateMeta.db"; + String fileName = PATH + "testUpdateMeta.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - byte[] bytesUpdate = new byte[]{2,3,4,5}; + byte[] bytesUpdate = new byte[]{2, 3, 4, 5}; var txContext = TransactionContext.empty(); var recoveryStorage = iItemStorage.getRecoveryStorage(); // 测试表中有 int xid = 0; List snaps = new ArrayList<>(); - recoveryStorage.begin(xid,snaps); - var uuid = iItemStorage.setMetadata(txContext,bytes); + recoveryStorage.begin(xid, snaps); + var uuid = iItemStorage.setMetadata(txContext, bytes); recoveryStorage.commit(xid); - assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); + Assert.assertTrue(Arrays.equals(bytes, iItemStorage.getMetadata())); // 测试表中没有头信息时更新 int xid2 = 1; - recoveryStorage.begin(xid2,snaps); - recoveryStorage.updateMeta(xid2,uuid,bytesUpdate); + recoveryStorage.begin(xid2, snaps); + recoveryStorage.updateMeta(xid2, uuid, bytesUpdate); recoveryStorage.commit(xid2); recoveryStorage.recovery(); - assertTrue(Arrays.equals(bytesUpdate,iItemStorage.getMetadata())); + Assert.assertTrue(Arrays.equals(bytesUpdate, iItemStorage.getMetadata())); } + @Test public void testUpdateMetaFail() throws PageException, LogException, FileException, IOException { - new File("test_gen_files/testUpdateMetaFail.db").delete(); - new File("test_gen_files/testUpdateMetaFail.db.log").delete(); - String fileName = "test_gen_files/testUpdateMetaFail.db"; + String fileName = PATH + "testUpdateMetaFail.db"; IItemStorage iItemStorage = ItemManager.fromFile(fileName); byte[] bytes = new byte[]{1, 2, 3, 4}; - byte[] bytesUpdate = new byte[]{2,3,4,5}; + byte[] bytesUpdate = new byte[]{2, 3, 4, 5}; var txContext = TransactionContext.empty(); var recoveryStorage = iItemStorage.getRecoveryStorage(); // 测试表中有 int xid = 0; List snaps = new ArrayList<>(); - recoveryStorage.begin(xid,snaps); - var uuid = iItemStorage.setMetadata(txContext,bytes); + recoveryStorage.begin(xid, snaps); + var uuid = iItemStorage.setMetadata(txContext, bytes); recoveryStorage.commit(xid); - assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); + Assert.assertTrue(Arrays.equals(bytes, iItemStorage.getMetadata())); // 测试事务失败 int xid2 = 1; - recoveryStorage.begin(xid2,snaps); - recoveryStorage.updateMeta(xid2,uuid,bytesUpdate); + recoveryStorage.begin(xid2, snaps); + recoveryStorage.updateMeta(xid2, uuid, bytesUpdate); recoveryStorage.rollback(xid2); recoveryStorage.recovery(); - assertTrue(Arrays.equals(bytes,iItemStorage.getMetadata())); - + Assert.assertTrue(Arrays.equals(bytes, iItemStorage.getMetadata())); } - - } diff --git a/testInsert.db b/testInsert.db deleted file mode 100644 index ca5ea44f24f65b3d8acaaf654c430de8c51b16e2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 40960 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd t0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0Rsm92L_-300961 From f31d3a7df83dac262659189c67311c18a4e7c753 Mon Sep 17 00:00:00 2001 From: KAAAsS Date: Sun, 17 Jan 2021 14:51:58 +0800 Subject: [PATCH 20/20] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E5=8F=AF=E5=8F=91?= =?UTF-8?q?=E7=8E=B0=E7=9A=84=E7=B3=BB=E5=88=97=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kaaass/rumbase/dataitem/ItemManager.java | 9 +- .../kaaass/rumbase/page/RumPageStorage.java | 3 +- .../rumbase/recovery/RecoveryManager.java | 13 +- .../rumbase/recovery/RecoveryStorage.java | 132 ++++++++++-------- .../recovery/exception/LogException.java | 4 + .../net/kaaass/rumbase/server/Server.java | 26 +++- .../kaaass/rumbase/table/TableManager.java | 52 +++++-- .../transaction/TransactionContextImpl.java | 2 + 8 files changed, 155 insertions(+), 86 deletions(-) diff --git a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java index 8c5148e..f86143d 100644 --- a/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java +++ b/src/main/java/net/kaaass/rumbase/dataitem/ItemManager.java @@ -7,7 +7,6 @@ import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.transaction.TransactionContext; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -37,7 +36,7 @@ public static IItemStorage fromFile(String fileName) throws FileException, PageE } } - public static IItemStorage fromFileWithoutLog(String fileName) throws FileException,PageException, PageCorruptedException { + public static IItemStorage fromFileWithoutLog(String fileName) throws FileException, PageException, PageCorruptedException { if (maps.containsKey(fileName)) { return maps.get(fileName); } else { @@ -69,10 +68,10 @@ public static IItemStorage createFile(TransactionContext txContext, String fileN } } - public static IItemStorage createFileWithoutLog(String fileName, byte[] metadata) throws FileException, PageException { - // 如果文件已经存在，那么就抛出文件已存在异常 + public static IItemStorage createFileWithoutLog(String fileName, byte[] metadata) throws FileException, PageException { + // 如果文件已经存在，那就返回 if (maps.containsKey(fileName)) { - throw new FileException(1); + return maps.get(fileName); } else { // 若文件不存在，则创建文件。 IItemStorage iItemStorage = ItemStorage.ofNewFileWithoutLog(fileName, metadata); diff --git a/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java b/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java index 1f20f82..1a827af 100644 --- a/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java +++ b/src/main/java/net/kaaass/rumbase/page/RumPageStorage.java @@ -29,7 +29,8 @@ public Page get(long pageId) { FileOutputStream out = new FileOutputStream(file); out.write(new byte[PageManager.PAGE_SIZE * 10]); } catch (IOException e) { - e.printStackTrace(); + log.error("创建文件失败", e); + System.exit(1); } } //文件会预留5页作为文件头 diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java index ae3b0b0..509fff8 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryManager.java @@ -1,5 +1,6 @@ package net.kaaass.rumbase.recovery; +import lombok.extern.slf4j.Slf4j; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; @@ -9,9 +10,11 @@ /** * 日志恢复的管理器，用来对每个数据库文件进行恢复 + * FIXME 需要同时在所有文件记录事务的发生 * * @author kaito */ +@Slf4j public class RecoveryManager { /** * 对某个数据库文件进行恢复 @@ -20,8 +23,14 @@ public class RecoveryManager { * @return 数据库日志管理器 */ public static void recovery(String fileName) throws LogException { - - RecoveryStorage.ofFile(fileName).recovery(); + IRecoveryStorage recoveryStorage; + try { + recoveryStorage = RecoveryStorage.ofFile(fileName); + } catch (LogException e) { + log.debug("恢复日志不存在，忽略该文件的恢复", e); + return; + } + recoveryStorage.recovery(); } public static IRecoveryStorage getRecoveryStorage(String fileName) throws LogException { diff --git a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java index d0c0f91..700a7e2 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java +++ b/src/main/java/net/kaaass/rumbase/recovery/RecoveryStorage.java @@ -3,10 +3,11 @@ import com.igormaznitsa.jbbp.JBBPParser; import com.igormaznitsa.jbbp.io.JBBPOut; import com.igormaznitsa.jbbp.mapper.Bin; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; import net.kaaass.rumbase.dataitem.IItemStorage; import net.kaaass.rumbase.dataitem.ItemManager; import net.kaaass.rumbase.dataitem.exception.UUIDException; -import net.kaaass.rumbase.page.Page; import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.page.exception.PageException; import net.kaaass.rumbase.recovery.exception.LogException; @@ -18,11 +19,12 @@ import java.util.Map; /** - * 日志记录相关内容 + * 日志记录相关内容 * * @author kaito */ -public class RecoveryStorage implements IRecoveryStorage { +@Slf4j +public class RecoveryStorage implements IRecoveryStorage { /** @@ -34,37 +36,39 @@ public class RecoveryStorage implements IRecoveryStorage { */ private String fileName; - public RecoveryStorage(IItemStorage itemStorage,String fileName) { + public RecoveryStorage(IItemStorage itemStorage, String fileName) { this.logStorage = itemStorage; this.fileName = fileName; } /** * 读取日志文件并且解析得到日志管理器 + * * @param fileName * @return 日志管理器 */ public static IRecoveryStorage ofFile(String fileName) throws LogException { try { var itemStorage = ItemManager.fromFileWithoutLog(fileName + ".log"); - return new RecoveryStorage(itemStorage,fileName); - }catch (FileException | PageException e){ + return new RecoveryStorage(itemStorage, fileName); + } catch (FileException | PageException e) { throw new LogException(10); } } /** * 创建日志文件 + * * @param fileName 文件名 * @return */ public static IRecoveryStorage ofNewFile(String fileName) throws LogException { try { var metadata = JBBPOut.BeginBin().Int(HEADER).End().toByteArray(); - var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log",metadata); - return new RecoveryStorage(itemStorage,fileName); - }catch (IOException | FileException | PageException e) { - throw new LogException(9); + var itemStorage = ItemManager.createFileWithoutLog(fileName + ".log", metadata); + return new RecoveryStorage(itemStorage, fileName); + } catch (IOException | FileException | PageException e) { + throw new LogException(9, e); } } @@ -75,13 +79,13 @@ public void begin(int xid, List snapshots) throws LogException { Byte(TX_BEGIN). Int(xid). Int(snapshots.size()); - for ( var i : snapshots){ + for (var i : snapshots) { jbbp = jbbp.Int(i); } var bytes = jbbp.End().toByteArray(); var uuid = logStorage.insertItemWithoutLog(bytes); logStorage.flush(uuid); - } catch (IOException | FileException e) { + } catch (IOException | FileException e) { throw new LogException(3); } } @@ -95,7 +99,7 @@ public void rollback(int xid) throws LogException { End().toByteArray(); var uuid = logStorage.insertItemWithoutLog(bytes); logStorage.flush(uuid); - }catch (FileException | IOException e){ + } catch (FileException | IOException e) { throw new LogException(4); } } @@ -109,7 +113,7 @@ public void commit(int xid) throws LogException { End().toByteArray(); var uuid = logStorage.insertItemWithoutLog(bytes); logStorage.flush(uuid); - }catch (FileException | IOException e){ + } catch (FileException | IOException e) { throw new LogException(5); } @@ -127,14 +131,14 @@ public void insert(int xid, long uuid, byte[] item) throws LogException { End().toByteArray(); var id = logStorage.insertItemWithoutLog(bytes); logStorage.flush(id); - }catch (FileException | IOException e){ + } catch (FileException | IOException e) { throw new LogException(6); } } @Override public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) throws LogException { - try{ + try { var bytes = JBBPOut.BeginBin(). Byte(UPDATE_FLAG). Int(xid). @@ -146,15 +150,15 @@ public void update(int xid, long uuid, byte[] itemBefore, byte[] itemAfter) thro End().toByteArray(); var id = logStorage.insertItemWithoutLog(bytes); logStorage.flush(id); - }catch (FileException | IOException e){ + } catch (FileException | IOException e) { throw new LogException(7); } } @Override - public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws LogException { + public void updateMeta(int xid, long beforeUuid, byte[] metadata) throws LogException { try { - var bytes =JBBPOut.BeginBin(). + var bytes = JBBPOut.BeginBin(). Byte(METADATA_UPDATE_FLAG). Int(xid). Long(beforeUuid). @@ -163,7 +167,7 @@ public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws LogExcep End().toByteArray(); var uuid = logStorage.insertItemWithoutLog(bytes); logStorage.flush(uuid); - }catch (FileException | IOException e){ + } catch (FileException | IOException e) { throw new LogException(8); } @@ -173,10 +177,10 @@ public void updateMeta(int xid, long beforeUuid,byte[] metadata) throws LogExcep public List getContent() { List logList = new ArrayList<>(); var maxPageId = logStorage.getMaxPageId(); - for (int i = 1;i <= maxPageId ; i ++){ + for (int i = 1; i <= maxPageId; i++) { var logs = logStorage.listItemByPageId(i); - for(var l : logs){ - if (l.length > 4){ + for (var l : logs) { + if (l.length > 4) { logList.add(l); } } @@ -185,15 +189,19 @@ public List getContent() { } @Override - public void recovery() throws LogException{ + public void recovery() throws LogException { try { var itemStorage = ItemManager.fromFile(this.fileName); + // TODO 判断是否进行恢复 + log.info("开始恢复文件 {}...", this.fileName); var logs = getContent(); var xidMaps = parseXid(logs); - for (var log : logs){ - parseLog(log,itemStorage,xidMaps); + log.debug("文件 {} 回滚事务：{}", this.fileName, xidMaps); + // 逐条回滚 + for (var log : logs) { + parseLog(log, itemStorage, xidMaps); } - }catch (FileException | PageException e){ + } catch (FileException | PageException e) { throw new LogException(11); } } @@ -201,18 +209,18 @@ public void recovery() throws LogException{ /** * 先进行一遍解析，得到所有已完成的事务和未完成的事务，来决定redo还是undo */ - private Map> parseXid(List logs) throws LogException { - List commitXids = new ArrayList<>(); + private Map> parseXid(List logs) throws LogException { + List commitXids = List.of(0); List abortXids = new ArrayList<>(); - for (var log : logs){ - Tx tx = null; + for (var binLog : logs) { + Tx tx; try { - tx = parseTx(log); + tx = parseTx(binLog); } catch (IOException e) { throw new LogException(1); } - switch (tx.type){ + switch (tx.type) { case TX_COMMIT: // 若是commit 则放入commitXid并将对应的xid移出abort Integer id = tx.xid; @@ -229,9 +237,9 @@ private Map> parseXid(List logs) throws LogExcep } } - Map> maps = new HashMap<>(); - maps.put('C',commitXids); - maps.put('A',abortXids); + Map> maps = new HashMap<>(); + maps.put('C', commitXids); + maps.put('A', abortXids); return maps; } @@ -248,7 +256,7 @@ private Tx parseTx(byte[] log) throws IOException { */ private byte parseType(byte[] log) throws IOException { var type = JBBPParser.prepare("byte type;").parse(log).mapTo(new Type()).type; - return type ; + return type; } /** @@ -272,16 +280,16 @@ private UpdateLog parseUpdate(byte[] log) throws IOException { private UpdateMetaLog parseUpdateMeta(byte[] log) throws IOException { var updateMeta = JBBPParser.prepare("byte type;int xid;long beforeUuid;" + "int length;byte[length] metadata;"). - parse(log).mapTo(new UpdateMetaLog()); + parse(log).mapTo(new UpdateMetaLog()); return updateMeta; } - private boolean checkCommit(int xid,Map> maps) throws LogException { - if (maps.get('C').contains(xid)){ + private boolean checkCommit(int xid, Map> maps) throws LogException { + if (maps.get('C').contains(xid)) { return true; - }else if (maps.get('A').contains(xid)){ + } else if (maps.get('A').contains(xid)) { return false; - }else { + } else { throw new LogException(2); } } @@ -289,14 +297,14 @@ private boolean checkCommit(int xid,Map> maps) throws Lo private void parseLog(byte[] log, IItemStorage itemStorage, Map> xidMaps) throws LogException { try { var type = parseType(log); - switch (type){ + switch (type) { case INSERT_FLAG: System.out.println("正在解析插入"); var insertLog = parseInsert(log); - if (checkCommit(insertLog.xid,xidMaps)){ + if (checkCommit(insertLog.xid, xidMaps)) { // 如果事务已经提交，则redo - itemStorage.insertItemWithUuid(insertLog.item,insertLog.uuid); - }else { + itemStorage.insertItemWithUuid(insertLog.item, insertLog.uuid); + } else { // 如果事务没有提交，则undo itemStorage.deleteUuid(insertLog.uuid); } @@ -304,17 +312,17 @@ private void parseLog(byte[] log, IItemStorage itemStorage, Map klazz) { return klazz == InsertLog.class ? new InsertLog() : null; } @@ -388,7 +398,7 @@ public Object newInstance(Class klazz) { /** * 更新的解析 */ - public static class UpdateLog{ + public static class UpdateLog { @Bin byte type; @Bin @@ -403,12 +413,13 @@ public static class UpdateLog{ int length2; @Bin byte[] itemAfter; + public Object newInstance(Class klazz) { return klazz == UpdateLog.class ? new UpdateLog() : null; } } - public static class UpdateMetaLog{ + public static class UpdateMetaLog { @Bin byte type; @Bin @@ -419,6 +430,7 @@ public static class UpdateMetaLog{ int length; @Bin byte[] metadata; + public Object newInstance(Class klazz) { return klazz == UpdateMetaLog.class ? new UpdateMetaLog() : null; } @@ -442,6 +454,4 @@ public Object newInstance(Class klazz) { final private static byte METADATA_UPDATE_FLAG = 'm'; - - } diff --git a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java index 6b6ae8a..bc44422 100644 --- a/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java +++ b/src/main/java/net/kaaass/rumbase/recovery/exception/LogException.java @@ -28,4 +28,8 @@ public class LogException extends RumbaseException { public LogException(int subId) { super(3001,subId,REASONS.get(subId)); } + + public LogException(int subId, Throwable e) { + super(3001,subId,REASONS.get(subId), e); + } } diff --git a/src/main/java/net/kaaass/rumbase/server/Server.java b/src/main/java/net/kaaass/rumbase/server/Server.java index d3783d9..ba6a9ee 100644 --- a/src/main/java/net/kaaass/rumbase/server/Server.java +++ b/src/main/java/net/kaaass/rumbase/server/Server.java @@ -8,6 +8,8 @@ import net.kaaass.rumbase.page.exception.FileException; import net.kaaass.rumbase.query.exception.ArgumentException; import net.kaaass.rumbase.record.exception.RecordNotFoundException; +import net.kaaass.rumbase.recovery.RecoveryManager; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.table.TableManager; import net.kaaass.rumbase.table.exception.TableConflictException; import net.kaaass.rumbase.table.exception.TableExistenceException; @@ -51,10 +53,14 @@ public class Server { */ public void prepare() { // 准备文件夹 - var tableFolder = new File("data/table/a"); - assert tableFolder.exists() || tableFolder.mkdirs(); - var indexFolder = new File("data/index/a"); - assert indexFolder.exists() || indexFolder.mkdirs(); + var tableFolder = new File("data/table/"); + if (!tableFolder.exists()) { + tableFolder.mkdirs(); + } + var indexFolder = new File("data/index/"); + if (!indexFolder.exists()) { + indexFolder.mkdirs(); + } // 初始化事务管理器 log.info("初始化事务管理器..."); try { @@ -63,15 +69,23 @@ public void prepare() { log.error("初始化事务管理器失败", e); System.exit(1); } + // 恢复表管理器 + try { + RecoveryManager.recovery("data/metadata.db"); + } catch (LogException e) { + log.error("无法恢复表管理器，数据可能损坏！", e); + System.exit(1); + } // 初始化表管理器 log.info("初始化表管理器..."); - // TODO 先恢复metadata try { tableManager = new TableManager(); - } catch (TableExistenceException | TableConflictException | RecordNotFoundException | ArgumentException | IndexAlreadyExistException e) { + } catch (IndexAlreadyExistException e) { log.error("初始化表管理器失败", e); System.exit(1); } + // 恢复、载入其他表 + tableManager.prepare(); // 初始化线程池 log.info("初始化线程池..."); var namedThreadFactory = Executors.defaultThreadFactory(); diff --git a/src/main/java/net/kaaass/rumbase/table/TableManager.java b/src/main/java/net/kaaass/rumbase/table/TableManager.java index ecb925a..b4ff1c8 100644 --- a/src/main/java/net/kaaass/rumbase/table/TableManager.java +++ b/src/main/java/net/kaaass/rumbase/table/TableManager.java @@ -1,11 +1,14 @@ package net.kaaass.rumbase.table; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; import net.kaaass.rumbase.index.exception.IndexAlreadyExistException; import net.kaaass.rumbase.query.exception.ArgumentException; import net.kaaass.rumbase.record.IRecordStorage; import net.kaaass.rumbase.record.RecordManager; import net.kaaass.rumbase.record.exception.RecordNotFoundException; +import net.kaaass.rumbase.recovery.RecoveryManager; +import net.kaaass.rumbase.recovery.exception.LogException; import net.kaaass.rumbase.table.exception.TableConflictException; import net.kaaass.rumbase.table.field.BaseField; import net.kaaass.rumbase.table.exception.TableExistenceException; @@ -25,7 +28,7 @@ * * @author @KveinAxel */ - +@Slf4j public class TableManager { static { @@ -68,11 +71,11 @@ public void abort(TransactionContext context) { context.rollback(); } - public TableManager() throws TableExistenceException, TableConflictException, RecordNotFoundException, ArgumentException, IndexAlreadyExistException { + public TableManager() throws IndexAlreadyExistException { load(); } - public void load() throws TableExistenceException, TableConflictException, RecordNotFoundException, ArgumentException, IndexAlreadyExistException { + public void load() throws IndexAlreadyExistException { var context = TransactionContext.empty(); Table metaTable; try { @@ -102,29 +105,56 @@ public void load() throws TableExistenceException, TableConflictException, Recor metaTable.persist(context); tableCache.put("metadata", metaTable); } - var data = metaTable.readAll(context); + } + + /** + * 启动服务器前进行的准备 + */ + public void prepare() { + var context = TransactionContext.empty(); + var metaTable = tableCache.get("metadata"); + List> data = null; + try { + data = metaTable.readAll(context); + } catch (TableExistenceException | TableConflictException | ArgumentException | RecordNotFoundException e) { + log.error("无法读入元数据表，数据可能损坏！", e); + System.exit(1); + } + // 载入所有已有表 var map = new HashMap(); data.forEach(row -> map.put((String) row.get(0), (String) row.get(1))); if (!map.containsKey("table_num")) { - metaTable.insert(context, new ArrayList<>(){{ - add("'table_num'"); - add("'0'"); - }}); + try { + metaTable.insert(context, new ArrayList<>(){{ + add("'table_num'"); + add("'0'"); + }}); + } catch (TableConflictException | TableExistenceException | ArgumentException e) { + log.error("无法初始化元数据表", e); + System.exit(1); + } map.put("table_num", "0"); } for (var item: map.entrySet()) { if (item.getKey().startsWith("tablePath$")) { var tableName = item.getKey().split("\\$")[1]; + var tablePath = item.getValue(); + // 恢复表 + try { + RecoveryManager.recovery(tablePath); + } catch (LogException e) { + log.error("无法恢复表 {} 于 {}，数据可能损坏！", tableName, tablePath, e); + System.exit(1); + } + // 读入表 var record = RecordManager.fromFile(item.getValue()); - recordPaths.add(item.getValue()); + recordPaths.add(tablePath); var table = Table.load(record); tableCache.put(tableName, table); } } - - } /** diff --git a/src/main/java/net/kaaass/rumbase/transaction/TransactionContextImpl.java b/src/main/java/net/kaaass/rumbase/transaction/TransactionContextImpl.java index bb803bd..6a0f04e 100644 --- a/src/main/java/net/kaaass/rumbase/transaction/TransactionContextImpl.java +++ b/src/main/java/net/kaaass/rumbase/transaction/TransactionContextImpl.java @@ -2,6 +2,8 @@ import lombok.Getter; import lombok.Setter; +import net.kaaass.rumbase.recovery.RecoveryManager; +import net.kaaass.rumbase.recovery.RecoveryStorage; import net.kaaass.rumbase.transaction.exception.DeadlockException; import net.kaaass.rumbase.transaction.lock.LockTable; import net.kaaass.rumbase.transaction.lock.LockTableImpl;