From 3f77f53eab5c28f5e291a36b1a39b93719c992e3 Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Wed, 14 Oct 2015 13:45:53 +0300 Subject: [PATCH 1/3] COUCHDB-769: Store attachments in external storage. This is initial implementation, supports OpenStack Swift and SoftLayer Object store --- src/chttpd_db.erl | 85 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 14 deletions(-) diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl index 552e5f8..8fed06f 100644 --- a/src/chttpd_db.erl +++ b/src/chttpd_db.erl @@ -407,7 +407,15 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req, true -> [all_or_nothing|Options]; _ -> Options end, - case fabric:update_docs(Db, Docs, Options2) of + couch_log:debug("chttpd_db: update_docs wellcome. Going to store attachment in external store",[]), + case fabric_attachments_handler:externalize_att(Db) of + "true" -> + NewDocs = fabric_attachments_handler:inline_att_store(Db,fabric:docs(Docs)); + "false" -> + couch_log:debug("Store inline attachmets in object store disabled",[]), + NewDocs = Docs + end, + case fabric:update_docs(Db, NewDocs, Options2) of {ok, Results} -> % output the results DocResults = lists:zipwith(fun update_doc_result_to_json/2, @@ -733,7 +741,17 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) -> NewDoc = Doc#doc{ atts = UpdatedAtts ++ OldAtts2 }, - case fabric:update_doc(Db, NewDoc, Options) of + couch_log:debug("chttpd_db: couchNew doc formed including new att: formed",[]), + lists:foreach(fun(X) -> couch_log:debug("~p~n ~p~n",[couch_att:fetch(data, X), couch_att:fetch(name, X)]) end, NewDoc#doc.atts), + couch_log:debug("chttpd_db: going to call attachment handler for base64 attachments",[]), + case fabric_attachments_handler:externalize_att(Db) of + "true" -> + NewNewDoc = fabric_attachments_handler:inline_att_store(Db, NewDoc); %store_single_document + "false" -> + couch_log:debug("Store inline attachmets in Swift disabled",[]), + NewNewDoc = NewDoc + end, + case fabric:update_doc(Db, NewNewDoc, Options) of {ok, NewRev} -> HttpCode = 201; {accepted, NewRev} -> @@ -1079,7 +1097,17 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa case [A || A <- Atts, couch_att:fetch(name, A) == FileName] of [] -> throw({not_found, "Document is missing attachment"}); - [Att] -> + [TmpAtt] -> + AttExternal = couch_att:fetch(att_external,TmpAtt), + couch_log:debug("chttpd_db: attachment stored in external store ~p~n",[AttExternal]), + case AttExternal of + "external" -> + Att = fabric_attachments_handler:inline_att_handler(get, Db, TmpAtt), + couch_log:debug("chtttpd_db: Got attachment from external store",[]); + _ -> + couch_log:debug("chttpd_db: Att is not stored in external store",[]), + Att = TmpAtt + end, [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch([type, encoding, disk_len, att_len, md5], Att), Refs = monitor_attachments(Att), try @@ -1127,12 +1155,20 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa % header we'll fall back to a chunked response. undefined end, - AttFun = case ReqAcceptsAttEnc of + AttFunTmp = case ReqAcceptsAttEnc of false -> fun couch_att:foldl_decode/3; true -> fun couch_att:foldl/3 end, + case AttExternal of + "external" -> + AttFun = couch_att:fetch(data,Att), + couch_log:debug("chtttpd_db: Got new attachment ",[]); + _ -> + couch_log:debug("chttpd_db: not external att",[]), + AttFun = AttFunTmp + end, chttpd:etag_respond( Req, Etag, @@ -1161,7 +1197,12 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa [] end, {ok, Resp} = start_response_length(Req, 200, Headers1, Len), - AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + case AttExternal of + "external" -> + send(Resp,couch_att:fetch(data, Att)), {ok, Resp}; + _ -> + AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + end end end end @@ -1189,7 +1230,6 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa undefined -> <<"application/octet-stream">>; CType -> list_to_binary(CType) end, - Data = fabric:att_receiver(Req, chttpd:body_length(Req)), ContentLen = case couch_httpd:header_value(Req,"Content-Length") of undefined -> undefined; Length -> list_to_integer(Length) @@ -1208,14 +1248,31 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa "Only gzip and identity content-encodings are supported" }) end, - [couch_att:new([ - {name, FileName}, - {type, MimeType}, - {data, Data}, - {att_len, ContentLen}, - {md5, get_md5_header(Req)}, - {encoding, Encoding} - ])] + case fabric_attachments_handler:externalize_att(Db) of + "true" -> + {Data, ExternalObjectSize, ExternalEtagMD5} = fabric_attachments_handler:normal_att_store(FileName,Db,ContentLen,MimeType,Req), + [couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, ContentLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding}, + {att_external_size,ExternalObjectSize}, + {att_external_md5,ExternalEtagMD5}, + {att_external,"external"} + ])]; + "false" -> + Data = fabric:att_receiver(Req, chttpd:body_length(Req)), + [couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, ContentLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding} + ])] + end end, Doc = case extract_header_rev(Req, chttpd:qs_value(Req, "rev")) of From 843e24f7c592616517b3129fffeb39bed405a00f Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Mon, 4 Jan 2016 10:12:40 +0200 Subject: [PATCH 2/3] improved code. based on the previous review --- src/chttpd_db.erl | 100 ++++++++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 47 deletions(-) diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl index 530e268..1fe2edd 100644 --- a/src/chttpd_db.erl +++ b/src/chttpd_db.erl @@ -407,12 +407,13 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req, true -> [all_or_nothing|Options]; _ -> Options end, - couch_log:debug("chttpd_db: update_docs wellcome. Going to store attachment in external store",[]), - case fabric_attachments_handler:externalize_att(Db) of - "true" -> - NewDocs = fabric_attachments_handler:inline_att_store(Db,fabric:docs(Docs)); - "false" -> - couch_log:debug("Store inline attachmets in object store disabled",[]), + case fabric_att_handler:external_store() of + true -> + couch_log:debug("store attachment externaly", []), + NewDocs = fabric_att_handler:att_store(Db, + fabric:docs(Docs)); + false -> + couch_log:debug("externalize attachment: disabled",[]), NewDocs = Docs end, case fabric:update_docs(Db, NewDocs, Options2) of @@ -730,15 +731,12 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) -> NewDoc = Doc#doc{ atts = UpdatedAtts ++ OldAtts2 }, - couch_log:debug("chttpd_db: couchNew doc formed including new att: formed",[]), - lists:foreach(fun(X) -> couch_log:debug("~p~n ~p~n",[couch_att:fetch(data, X), couch_att:fetch(name, X)]) end, NewDoc#doc.atts), - couch_log:debug("chttpd_db: going to call attachment handler for base64 attachments",[]), - case fabric_attachments_handler:externalize_att(Db) of - "true" -> - NewNewDoc = fabric_attachments_handler:inline_att_store(Db, NewDoc); %store_single_document - "false" -> + NewNewDoc = case fabric_att_handler:external_store() of + true -> + fabric_att_handler:att_store(Db, NewDoc); %store_single_document + false -> couch_log:debug("Store inline attachmets in Swift disabled",[]), - NewNewDoc = NewDoc + NewDoc end, case fabric:update_doc(Db, NewNewDoc, Options) of {ok, NewRev} -> @@ -1083,15 +1081,14 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa [] -> throw({not_found, "Document is missing attachment"}); [TmpAtt] -> - AttExternal = couch_att:fetch(att_external,TmpAtt), - couch_log:debug("chttpd_db: attachment stored in external store ~p~n",[AttExternal]), - case AttExternal of - "external" -> - Att = fabric_attachments_handler:inline_att_handler(get, Db, TmpAtt), - couch_log:debug("chtttpd_db: Got attachment from external store",[]); + ExtStoreID = couch_att:fetch(att_extstore_id,TmpAtt), + case ExtStoreID of + undefined -> + couch_log:debug("chttpd_db: Att is not stored in the external store ~p~n",[ExtStoreID]), + Att = TmpAtt; _ -> - couch_log:debug("chttpd_db: Att is not stored in external store",[]), - Att = TmpAtt + Att = fabric_att_handler:att(get, Db, TmpAtt), + couch_log:debug("chtttpd_db: Got attachment from the external store ~p~n",[ExtStoreID]) end, [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch([type, encoding, disk_len, att_len, md5], Att), Refs = monitor_attachments(Att), @@ -1146,13 +1143,11 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa true -> fun couch_att:foldl/3 end, - case AttExternal of - "external" -> - AttFun = couch_att:fetch(data,Att), - couch_log:debug("chtttpd_db: Got new attachment ",[]); + case ExtStoreID of + undefined -> + AttFun = AttFunTmp; _ -> - couch_log:debug("chttpd_db: not external att",[]), - AttFun = AttFunTmp + AttFun = couch_att:fetch(data,Att) end, chttpd:etag_respond( Req, @@ -1182,11 +1177,11 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa [] end, {ok, Resp} = start_response_length(Req, 200, Headers1, Len), - case AttExternal of - "external" -> - send(Resp,couch_att:fetch(data, Att)), {ok, Resp}; + case ExtStoreID of + undefined -> + AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}); _ -> - AttFun(Att, fun(Seg, _) -> send(Resp, Seg) end, {ok, Resp}) + send(Resp,couch_att:fetch(data, Att)), {ok, Resp} end end end @@ -1233,21 +1228,32 @@ db_attachment_req(#httpd{method=Method, user_ctx=Ctx}=Req, Db, DocId, FileNamePa "Only gzip and identity content-encodings are supported" }) end, - case fabric_attachments_handler:externalize_att(Db) of - "true" -> - {Data, ExternalObjectSize, ExternalEtagMD5} = fabric_attachments_handler:normal_att_store(FileName,Db,ContentLen,MimeType,Req), - [couch_att:new([ - {name, FileName}, - {type, MimeType}, - {data, Data}, - {att_len, ContentLen}, - {md5, get_md5_header(Req)}, - {encoding, Encoding}, - {att_external_size,ExternalObjectSize}, - {att_external_md5,ExternalEtagMD5}, - {att_external,"external"} - ])]; - "false" -> + case fabric_att_handler:external_store() of + true -> + case fabric_att_handler:att_store(FileName,Db,ContentLen,MimeType,Req) of + {ok,{Data, ExternalObjectSize, ExternalEtagMD5, StoreID}} -> + [couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, ContentLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding}, + {att_extstore_size,ExternalObjectSize}, + {att_extstore_md5,ExternalEtagMD5}, + {att_extstore_id,StoreID} + ])]; + {error, Data} -> + [couch_att:new([ + {name, FileName}, + {type, MimeType}, + {data, Data}, + {att_len, ContentLen}, + {md5, get_md5_header(Req)}, + {encoding, Encoding} + ])] + end; + false -> Data = fabric:att_receiver(Req, chttpd:body_length(Req)), [couch_att:new([ {name, FileName}, From fd55a03efdc2414fc4e5dcefa8d8a26b402dafcf Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Wed, 20 Jan 2016 09:23:06 +0200 Subject: [PATCH 3/3] code modification based on the recent reviews --- src/chttpd_db.erl | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/chttpd_db.erl b/src/chttpd_db.erl index 1fe2edd..a5e310a 100644 --- a/src/chttpd_db.erl +++ b/src/chttpd_db.erl @@ -407,14 +407,11 @@ db_req(#httpd{method='POST',path_parts=[_,<<"_bulk_docs">>], user_ctx=Ctx}=Req, true -> [all_or_nothing|Options]; _ -> Options end, - case fabric_att_handler:external_store() of + NewDocs = case fabric_att_handler:external_store() of true -> - couch_log:debug("store attachment externaly", []), - NewDocs = fabric_att_handler:att_store(Db, - fabric:docs(Docs)); + fabric_att_handler:att_store(Db, fabric:docs(Docs)); false -> - couch_log:debug("externalize attachment: disabled",[]), - NewDocs = Docs + Docs end, case fabric:update_docs(Db, NewDocs, Options2) of {ok, Results} -> @@ -735,15 +732,14 @@ db_doc_req(#httpd{method='POST', user_ctx=Ctx}=Req, Db, DocId) -> true -> fabric_att_handler:att_store(Db, NewDoc); %store_single_document false -> - couch_log:debug("Store inline attachmets in Swift disabled",[]), NewDoc end, case fabric:update_doc(Db, NewNewDoc, Options) of - {ok, NewRev} -> - HttpCode = 201; - {accepted, NewRev} -> - HttpCode = 202 - end, + {ok, NewRev} -> + HttpCode = 201; + {accepted, NewRev} -> + HttpCode = 202 + end, send_json(Req, HttpCode, [{"Etag", "\"" ++ ?b2l(couch_doc:rev_to_str(NewRev)) ++ "\""}], {[ {ok, true}, {id, DocId}, @@ -1084,11 +1080,15 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa ExtStoreID = couch_att:fetch(att_extstore_id,TmpAtt), case ExtStoreID of undefined -> - couch_log:debug("chttpd_db: Att is not stored in the external store ~p~n",[ExtStoreID]), Att = TmpAtt; _ -> - Att = fabric_att_handler:att(get, Db, TmpAtt), - couch_log:debug("chtttpd_db: Got attachment from the external store ~p~n",[ExtStoreID]) + Att = case fabric_att_handler:att_get(Db, TmpAtt) of + {ok, Att1} -> + couch_log:info("chtttpd_db: Got attachment from the external store ~p~n",[ExtStoreID]), + Att1; + {error, msg} -> + throw({error, msg}) + end end, [Type, Enc, DiskLen, AttLen, Md5] = couch_att:fetch([type, encoding, disk_len, att_len, md5], Att), Refs = monitor_attachments(Att), @@ -1137,18 +1137,12 @@ db_attachment_req(#httpd{method='GET',mochi_req=MochiReq}=Req, Db, DocId, FileNa % header we'll fall back to a chunked response. undefined end, - AttFunTmp = case ReqAcceptsAttEnc of + AttFun = case ReqAcceptsAttEnc of false -> fun couch_att:foldl_decode/3; true -> fun couch_att:foldl/3 end, - case ExtStoreID of - undefined -> - AttFun = AttFunTmp; - _ -> - AttFun = couch_att:fetch(data,Att) - end, chttpd:etag_respond( Req, Etag,