Skip to content

feat: add getActivePiecesByCursor for gas-efficient pagination #246

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rvagg merged 2 commits into from
Feb 3, 2026
Merged

feat: add getActivePiecesByCursor for gas-efficient pagination #246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3f4800e
feat: add getActivePiecesByCursor for O(limit) pagination
Chaitu-Tatipamula Jan 29, 2026
4f1999a
refactor: use assembly truncation instead of copy loops, fixes typos
Chaitu-Tatipamula Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 75 additions & 6 deletions src/PDPVerifier.sol
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,8 @@ contract PDPVerifier is Initializable, UUPSUpgradeable, OwnableUpgradeable {

/**
* @notice Returns active pieces (non-zero leaf count) for a data set with pagination
* @dev WARNING: This function has O(offset) gas complexity because it always iterates from
* piece ID 0. For large data sets with high offsets, consider using getActivePiecesByCursor()
* @param setId The data set ID
* @param offset Starting index for pagination (0-based)
* @param limit Maximum number of pieces to return
Expand Down Expand Up @@ -377,12 +379,11 @@ contract PDPVerifier is Initializable, UUPSUpgradeable, OwnableUpgradeable {
return (new Cids.Cid[](0), new uint256[](0), false);
} else if (resultIndex < limit) {
// Found fewer items than limit - need to resize arrays
pieces = new Cids.Cid[](resultIndex);
pieceIds = new uint256[](resultIndex);

for (uint256 i = 0; i < resultIndex; i++) {
pieces[i] = tempPieces[i];
pieceIds[i] = tempPieceIds[i];
pieces = tempPieces;
pieceIds = tempPieceIds;
assembly ("memory-safe") {
mstore(pieces, resultIndex)
mstore(pieceIds, resultIndex)
}
} else {
// Found exactly limit items - use temp arrays directly
Expand All @@ -391,6 +392,74 @@ contract PDPVerifier is Initializable, UUPSUpgradeable, OwnableUpgradeable {
}
}

/**
* @notice Returns active pieces using cursor-based pagination for O(limit) gas complexity
* @dev This function is more gas-efficient than getActivePieces() for large data sets because
* it starts iteration from startPieceId instead of always from 0. Use this for paginating
* through large data sets.
* @param setId The data set ID
* @param startPieceId The piece ID to start from (use 0 for first page, then last returned pieceId + 1)
* @param limit Maximum number of pieces to return
* @return pieces Array of active piece CIDs
* @return pieceIds Array of corresponding piece IDs
* @return hasMore True if there are more pieces beyond this page
*/
function getActivePiecesByCursor(uint256 setId, uint256 startPieceId, uint256 limit)
public
view
returns (Cids.Cid[] memory pieces, uint256[] memory pieceIds, bool hasMore)
{
require(dataSetLive(setId), "Data set not live");
require(limit > 0, "Limit must be greater than 0");

uint256 maxPieceId = nextPieceId[setId];

// if startPieceId is beyond all pieces, return empty
if (startPieceId >= maxPieceId) {
return (new Cids.Cid[](0), new uint256[](0), false);
}

// Over-allocate arrays to limit size
Cids.Cid[] memory tempPieces = new Cids.Cid[](limit);
uint256[] memory tempPieceIds = new uint256[](limit);
uint256 resultIndex = 0;

// Start from startPieceId and collect up to limit
for (uint256 i = startPieceId; i < maxPieceId && resultIndex < limit; i++) {
if (pieceLeafCounts[setId][i] > 0) {
tempPieces[resultIndex] = pieceCids[setId][i];
tempPieceIds[resultIndex] = i;
resultIndex++;
}
}

// Check if there are more active pieces after last one we found
if (resultIndex > 0) {
uint256 lastFound = tempPieceIds[resultIndex - 1];
for (uint256 i = lastFound + 1; i < maxPieceId; i++) {
if (pieceLeafCounts[setId][i] > 0) {
hasMore = true;
break;
}
}
}

// Handle case where we found fewer items than limit
if (resultIndex == 0) {
return (new Cids.Cid[](0), new uint256[](0), false);
} else if (resultIndex < limit) {
pieces = tempPieces;
pieceIds = tempPieceIds;
assembly ("memory-safe") {
mstore(pieces, resultIndex)
mstore(pieceIds, resultIndex)
}
} else {
pieces = tempPieces;
pieceIds = tempPieceIds;
}
}

// storage provider proposes new storage provider. If the storage provider proposes themself delete any outstanding proposed storage provider
function proposeDataSetStorageProvider(uint256 setId, address newStorageProvider) public {
require(dataSetLive(setId), "Data set not live");
Expand Down
110 changes: 110 additions & 0 deletions test/PDPVerifier.t.sol
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1132,6 +1132,116 @@ contract PDPVerifierPaginationTest is MockFVMTest, PieceHelper {

assertEq(totalRetrieved, 100, "Should have retrieved all 100 pieces");
}

function testGetActivePiecesByCursorBasic() public {
uint256 setId = pdpVerifier.addPieces{value: PDPFees.sybilFee()}(
NEW_DATA_SET_SENTINEL, address(listener), new Cids.Cid[](0), abi.encode(empty, empty)
);

// Add 10 pieces
Cids.Cid[] memory testPieces = new Cids.Cid[](10);
for (uint256 i = 0; i < 10; i++) {
testPieces[i] = makeSamplePiece(1024 / 32);
}
pdpVerifier.addPieces(setId, address(0), testPieces, empty);

// Test first page
(Cids.Cid[] memory pieces1, uint256[] memory ids1, bool hasMore1) =
pdpVerifier.getActivePiecesByCursor(setId, 0, 5);
assertEq(pieces1.length, 5, "Should return 5 pieces");
assertEq(ids1[0], 0, "First ID should be 0");
assertEq(ids1[4], 4, "Last ID should be 4");
assertEq(hasMore1, true, "Should have more");

// Test second page using cursor (last ID + 1)
(Cids.Cid[] memory pieces2, uint256[] memory ids2, bool hasMore2) =
pdpVerifier.getActivePiecesByCursor(setId, ids1[4] + 1, 5);
assertEq(pieces2.length, 5, "Should return 5 pieces");
assertEq(ids2[0], 5, "First ID should be 5");
assertEq(ids2[4], 9, "Last ID should be 9");
assertEq(hasMore2, false, "Should not have more");
}

function testGetActivePiecesByCursorWithDeleted() public {
uint256 setId = pdpVerifier.addPieces{value: PDPFees.sybilFee()}(
NEW_DATA_SET_SENTINEL, address(listener), new Cids.Cid[](0), abi.encode(empty, empty)
);

// Add 10 pieces
Cids.Cid[] memory testPieces = new Cids.Cid[](10);
for (uint256 i = 0; i < 10; i++) {
testPieces[i] = makeSamplePiece(1024 / 32);
}
pdpVerifier.addPieces(setId, address(0), testPieces, empty);

// Delete pieces 2, 4, 6
uint256[] memory toRemove = new uint256[](3);
toRemove[0] = 2;
toRemove[1] = 4;
toRemove[2] = 6;
pdpVerifier.schedulePieceDeletions(setId, toRemove, empty);
uint256 challengeFinality = pdpVerifier.getChallengeFinality();
pdpVerifier.nextProvingPeriod(setId, vm.getBlockNumber() + challengeFinality, empty);

// Cursor at piece 3 should skip deleted pieces and return 3, 5, 7, 8, 9
(Cids.Cid[] memory pieces, uint256[] memory ids, bool hasMore) =
pdpVerifier.getActivePiecesByCursor(setId, 3, 10);

assertEq(pieces.length, 5, "Should return 5 active pieces (3,5,7,8,9)");
assertEq(ids[0], 3, "First should be 3");
assertEq(ids[1], 5, "Second should be 5 (skipped 4)");
assertEq(ids[2], 7, "Third should be 7 (skipped 6)");
assertEq(ids[3], 8, "Fourth should be 8");
assertEq(ids[4], 9, "Fifth should be 9");
assertEq(hasMore, false, "No more pieces");
}

function testGetActivePiecesByCursorBeyondRange() public {
uint256 setId = pdpVerifier.addPieces{value: PDPFees.sybilFee()}(
NEW_DATA_SET_SENTINEL, address(listener), new Cids.Cid[](0), abi.encode(empty, empty)
);

Cids.Cid[] memory testPieces = new Cids.Cid[](5);
for (uint256 i = 0; i < 5; i++) {
testPieces[i] = makeSamplePiece(1024 / 32);
}
pdpVerifier.addPieces(setId, address(0), testPieces, empty);

// Cursor beyond all pieces
(Cids.Cid[] memory pieces, uint256[] memory ids, bool hasMore) =
pdpVerifier.getActivePiecesByCursor(setId, 100, 10);

assertEq(pieces.length, 0, "Should return empty");
assertEq(ids.length, 0, "Should return empty IDs");
assertEq(hasMore, false, "No more pieces");
}

function testGetActivePiecesByCursorEmpty() public {
uint256 setId = pdpVerifier.addPieces{value: PDPFees.sybilFee()}(
NEW_DATA_SET_SENTINEL, address(listener), new Cids.Cid[](0), abi.encode(empty, empty)
);

(Cids.Cid[] memory pieces, uint256[] memory ids, bool hasMore) =
pdpVerifier.getActivePiecesByCursor(setId, 0, 10);

assertEq(pieces.length, 0, "Should return empty for empty dataset");
assertEq(ids.length, 0, "Should return empty IDs");
assertEq(hasMore, false, "No more pieces");
}

function testGetActivePiecesByCursorNotLive() public {
vm.expectRevert("Data set not live");
pdpVerifier.getActivePiecesByCursor(999, 0, 10);
}

function testGetActivePiecesByCursorZeroLimit() public {
uint256 setId = pdpVerifier.addPieces{value: PDPFees.sybilFee()}(
NEW_DATA_SET_SENTINEL, address(listener), new Cids.Cid[](0), abi.encode(empty, empty)
);

vm.expectRevert("Limit must be greater than 0");
pdpVerifier.getActivePiecesByCursor(setId, 0, 0);
}
}

// TestingRecordKeeperService is a PDPListener that allows any amount of proof challenges
Expand Down