From 77c27b4290c57d22d8d99cb4c87504e5922bc0e3 Mon Sep 17 00:00:00 2001 From: "Paul J. Davis" Date: Wed, 10 Feb 2016 18:14:07 -0600 Subject: [PATCH 1/7] Update API for use with pluggable storage engines --- src/couch_index_updater.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index ad48f40..bf31460 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -166,7 +166,7 @@ update(Idx, Mod, IdxState) -> end end, - Proc = fun(DocInfo, _, {IdxStateAcc, _}) -> + Proc = fun(DocInfo, {IdxStateAcc, _}) -> case CommittedOnly and (GetSeq(DocInfo) > DbCommittedSeq) of true -> {stop, {IdxStateAcc, false}}; @@ -180,7 +180,7 @@ update(Idx, Mod, IdxState) -> {ok, InitIdxState} = Mod:start_update(Idx, PurgedIdxState, NumChanges), Acc0 = {InitIdxState, true}, - {ok, _, Acc} = couch_db:enum_docs_since(Db, CurrSeq, Proc, Acc0, []), + {ok, Acc} = couch_db:fold_changes(Db, CurrSeq, Proc, Acc0, []), {ProcIdxSt, SendLast} = Acc, % If we didn't bail due to hitting the last committed seq we need @@ -198,7 +198,7 @@ update(Idx, Mod, IdxState) -> purge_index(Db, Mod, IdxState) -> - DbPurgeSeq = couch_db:get_purge_seq(Db), + {ok, DbPurgeSeq} = couch_db:get_purge_seq(Db), IdxPurgeSeq = Mod:get(purge_seq, IdxState), if DbPurgeSeq == IdxPurgeSeq -> From 53485acc32aa2599ba84776ebdd21b6c6d6fbe0d Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Mon, 15 Aug 2016 09:37:05 -0400 Subject: [PATCH 2/7] Update couch_index for new purge API Wrap the current purge operation into a fold over purge_tree BugzID: 68276 --- src/couch_index_updater.erl | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index bf31460..a50cf1e 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -201,11 +201,19 @@ purge_index(Db, Mod, IdxState) -> {ok, DbPurgeSeq} = couch_db:get_purge_seq(Db), IdxPurgeSeq = Mod:get(purge_seq, IdxState), if - DbPurgeSeq == IdxPurgeSeq -> + IdxPurgeSeq == DbPurgeSeq -> {ok, IdxState}; - DbPurgeSeq == IdxPurgeSeq + 1 -> - {ok, PurgedIdRevs} = couch_db:get_last_purged(Db), - Mod:purge(Db, DbPurgeSeq, PurgedIdRevs, IdxState); true -> - reset + {ok, DbOldestPurgeSeq} = couch_db:get_oldest_purge_seq(Db), + if IdxPurgeSeq >= DbOldestPurgeSeq -> + FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> + {ok, [ {PurgeSeq, Id, Revs} | Acc]} + end, + {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, IdxPurgeSeq, FoldFun, [], []), + lists:foldr(fun({PSeq, Id, Revs}, State) -> + Mod:purge(Db, PSeq, [{Id, Revs}], State) end, + IdxState, PurgeSeqIdRevs); + true -> + reset + end end. From ee1bd92955ed78e0d93a5adfccc50938f2980bdd Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Wed, 19 Oct 2016 06:43:28 -0400 Subject: [PATCH 3/7] Update couch_index for new purge API Wrap the current purge operation into a fold over purge_tree BugzID: 68276 --- src/couch_index_updater.erl | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index a50cf1e..6acc775 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -207,12 +207,14 @@ purge_index(Db, Mod, IdxState) -> {ok, DbOldestPurgeSeq} = couch_db:get_oldest_purge_seq(Db), if IdxPurgeSeq >= DbOldestPurgeSeq -> FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> - {ok, [ {PurgeSeq, Id, Revs} | Acc]} + {ok, [{PurgeSeq, Id, Revs} | Acc]} end, - {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, IdxPurgeSeq, FoldFun, [], []), - lists:foldr(fun({PSeq, Id, Revs}, State) -> - Mod:purge(Db, PSeq, [{Id, Revs}], State) end, - IdxState, PurgeSeqIdRevs); + {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, + IdxPurgeSeq, FoldFun, [], []), + % using foldr to avoid unnecessary lists:reverse/1 call + lists:foldr(fun({PSeq, Id, Revs}, StateAcc) -> + Mod:purge(Db, PSeq, [{Id, Revs}], StateAcc) + end, IdxState, PurgeSeqIdRevs); true -> reset end From b7fba8739a76c62684e0fc32ac038feefe1b278d Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Fri, 4 Nov 2016 08:00:51 -0400 Subject: [PATCH 4/7] Correct couch_index_updater:purge_index function BugzID: 68276 --- src/couch_index_updater.erl | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index 6acc775..a40d61a 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -205,16 +205,20 @@ purge_index(Db, Mod, IdxState) -> {ok, IdxState}; true -> {ok, DbOldestPurgeSeq} = couch_db:get_oldest_purge_seq(Db), - if IdxPurgeSeq >= DbOldestPurgeSeq -> + % increase by 1, since we need to collect purges starting from IdxPurgeSeq+1 + if (IdxPurgeSeq + 1) >= DbOldestPurgeSeq -> FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> {ok, [{PurgeSeq, Id, Revs} | Acc]} end, {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, IdxPurgeSeq, FoldFun, [], []), % using foldr to avoid unnecessary lists:reverse/1 call - lists:foldr(fun({PSeq, Id, Revs}, StateAcc) -> - Mod:purge(Db, PSeq, [{Id, Revs}], StateAcc) - end, IdxState, PurgeSeqIdRevs); + NewStateAcc = lists:foldr(fun({PSeq, Id, Revs}, StateAcc0) -> + {ok, StateAcc} = Mod:purge(Db, PSeq, + [{Id, Revs}], StateAcc0), + StateAcc + end, IdxState, PurgeSeqIdRevs), + {ok, NewStateAcc}; true -> reset end From 5b835ab23cc554421fcdffe3b05c26a4e687fbb3 Mon Sep 17 00:00:00 2001 From: Mayya Sharipova Date: Thu, 8 Dec 2016 15:49:00 -0500 Subject: [PATCH 5/7] couch_db:get_purge_seq/1 returns Seq (not {ok, Seq}) BugzID: 68276 --- src/couch_index_updater.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index a40d61a..a5b2f26 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -198,7 +198,7 @@ update(Idx, Mod, IdxState) -> purge_index(Db, Mod, IdxState) -> - {ok, DbPurgeSeq} = couch_db:get_purge_seq(Db), + DbPurgeSeq = couch_db:get_purge_seq(Db), IdxPurgeSeq = Mod:get(purge_seq, IdxState), if IdxPurgeSeq == DbPurgeSeq -> From cdb04bf7ee37026b3c150fe9758e8cd44d07cbea Mon Sep 17 00:00:00 2001 From: jiangphcn Date: Thu, 16 Feb 2017 15:43:27 +0800 Subject: [PATCH 6/7] Update couch_index for clustered purge improvement - disable reset for impossible overflow of purge tree - update local purge document after every bath purge Bugzid: 68276 --- src/couch_index_updater.erl | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index a5b2f26..e63cc89 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -133,10 +133,7 @@ update(Idx, Mod, IdxState) -> DbUpdateSeq = couch_db:get_update_seq(Db), DbCommittedSeq = couch_db:get_committed_update_seq(Db), - PurgedIdxState = case purge_index(Db, Mod, IdxState) of - {ok, IdxState0} -> IdxState0; - reset -> exit({reset, self()}) - end, + {ok, PurgedIdxState} = purge_index(Db, Mod, IdxState), NumChanges = couch_db:count_changes_since(Db, CurrSeq), @@ -204,22 +201,17 @@ purge_index(Db, Mod, IdxState) -> IdxPurgeSeq == DbPurgeSeq -> {ok, IdxState}; true -> - {ok, DbOldestPurgeSeq} = couch_db:get_oldest_purge_seq(Db), - % increase by 1, since we need to collect purges starting from IdxPurgeSeq+1 - if (IdxPurgeSeq + 1) >= DbOldestPurgeSeq -> - FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> - {ok, [{PurgeSeq, Id, Revs} | Acc]} - end, - {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, - IdxPurgeSeq, FoldFun, [], []), - % using foldr to avoid unnecessary lists:reverse/1 call - NewStateAcc = lists:foldr(fun({PSeq, Id, Revs}, StateAcc0) -> - {ok, StateAcc} = Mod:purge(Db, PSeq, - [{Id, Revs}], StateAcc0), - StateAcc - end, IdxState, PurgeSeqIdRevs), - {ok, NewStateAcc}; - true -> - reset - end + FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> + {ok, [{PurgeSeq, Id, Revs} | Acc]} + end, + {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, + IdxPurgeSeq, FoldFun, [], []), + % using foldr to avoid unnecessary lists:reverse/1 call + NewStateAcc = lists:foldr(fun({PSeq, Id, Revs}, StateAcc0) -> + {ok, StateAcc} = Mod:purge(Db, PSeq, + [{Id, Revs}], StateAcc0), + StateAcc + end, IdxState, PurgeSeqIdRevs), + Mod:update_local_purge_doc(Db, NewStateAcc), + {ok, NewStateAcc} end. From 0720c7e8e8063fc8a5b495661179912c7c07c07b Mon Sep 17 00:00:00 2001 From: jiangphcn Date: Mon, 27 Feb 2017 21:27:24 +0800 Subject: [PATCH 7/7] Update couch_index for clustered purge - use couch_db:fold_purged_docs/5 with UUID Bugzid: 68276 --- src/couch_index_updater.erl | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/src/couch_index_updater.erl b/src/couch_index_updater.erl index e63cc89..0ba2b09 100644 --- a/src/couch_index_updater.erl +++ b/src/couch_index_updater.erl @@ -201,17 +201,13 @@ purge_index(Db, Mod, IdxState) -> IdxPurgeSeq == DbPurgeSeq -> {ok, IdxState}; true -> - FoldFun = fun(PurgeSeq, {Id, Revs}, Acc) -> - {ok, [{PurgeSeq, Id, Revs} | Acc]} - end, - {ok, PurgeSeqIdRevs} = couch_db:fold_purged_docs(Db, - IdxPurgeSeq, FoldFun, [], []), - % using foldr to avoid unnecessary lists:reverse/1 call - NewStateAcc = lists:foldr(fun({PSeq, Id, Revs}, StateAcc0) -> - {ok, StateAcc} = Mod:purge(Db, PSeq, - [{Id, Revs}], StateAcc0), + FoldFun = fun({PurgeSeq, _UUId, Id, Revs}, Acc) -> + {ok, StateAcc} = Mod:purge(Db, PurgeSeq, + [{Id, Revs}], Acc), StateAcc - end, IdxState, PurgeSeqIdRevs), + end, + {ok, NewStateAcc} = couch_db:fold_purged_docs(Db, + IdxPurgeSeq, FoldFun, IdxState, []), Mod:update_local_purge_doc(Db, NewStateAcc), {ok, NewStateAcc} end.