From d029d3ae64a047131b022f52126a8bf7f765f823 Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Wed, 14 Oct 2015 14:02:40 +0300 Subject: [PATCH 1/3] COUCHDB-769: Store attachments in external storage. This is initial implementation, supports OpenStack Swift and SoftLayer Object store --- src/fabric_attachments_handler.erl | 333 +++++++++++++++++++++++++++++ src/fabric_db_create.erl | 15 ++ 2 files changed, 348 insertions(+) create mode 100644 src/fabric_attachments_handler.erl diff --git a/src/fabric_attachments_handler.erl b/src/fabric_attachments_handler.erl new file mode 100644 index 0000000..c4aab32 --- /dev/null +++ b/src/fabric_attachments_handler.erl @@ -0,0 +1,333 @@ +%% @author gilv +%% @doc @todo Add description to fabric_swift_handler. + + +-module(fabric_attachments_handler). + +-include_lib("fabric/include/fabric.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]). + + +%% ==================================================================== +%% External API functions. I try to keep them as general as possible, +%% without correlation to specific object store that might be internally used. +%% ==================================================================== + +normal_att_store(FileName,Db,ContentLen,MimeType,Req) -> + ContainerName = container_name(Db), + couch_log:debug("Standard attachment handler",[]), + couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]), + %Bad implementation - no chunk reader. All kept in the memory. Should be fixed. + Data = couch_httpd:recv(Req, ContentLen), + case swift_put_object(ContainerName, FileName, MimeType, [], Data) of + {ok,{201,NewUrl}} -> + couch_log:debug("Created. Swift response code is 201 ~n",[]), + {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName), + {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5}; + {_,{Code,_}} -> + couch_log:debug("Swift response code is ~p~n",[]), + {Data, -1} + end. + +inline_att_store(Db, Docs) -> + DbName = container_name(Db), + couch_log:debug("Store inline base64 encoded attachment ~n",[]), + if + is_list(Docs) -> + couch_log:debug("Going to handle document list",[]), + DocsArray = Docs; + true -> + couch_log:debug("Going to handle single document",[]), + DocsArray = [Docs] + end, + [#doc{atts=Atts0} = Doc | Rest] = DocsArray, + if + length(Atts0) > 0 -> + couch_log:debug("Att length is larger than 0",[]), + Atts = [att_processor(DbName,Att) || Att <- Atts0], + if + is_list(Docs) -> + [Doc#doc{atts = Atts} | Rest]; + true -> + Doc#doc{atts = Atts} + end; + true -> + couch_log:debug("No attachments to handle",[]), + Docs + end. + +inline_att_handler(get, Db, Att) -> + DbName = container_name(Db), + couch_log:debug("Retrieve attachment",[]), + [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att), + couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]), + NewData = swift_get_object(DbName, Name), + NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att), + NewAtt; + +inline_att_handler(delete,Db, Att) -> + DbName = container_name(Db), + couch_log:debug("Delete attachment ~p~n",[]). + +container_handler(create,DbName) -> + couch_log:debug("Create container ~p~n",[DbName]), + case swift_create_container(DbName) of + {ok,{201,Container}} -> + couch_log:debug("Container ~p created succesfully ~n",[Container]), + {ok,Container}; + {error,_} -> + couch_log:debug("Container ~p creation failed ~n",[DbName]), + {error,DbName} + end; +container_handler(get,DbName) -> + couch_log:debug("Get container ~p~n",[DbName]); +container_handler(delete,DbName)-> + couch_log:debug("Delete container ~p~n",[DbName]), + swift_delete_container(DbName). + +externalize_att(Db) -> + Res = config:get("swift","attachments_offload","false"), + Res. + +%% ==================================================================== +%% Internal general functions +%% ==================================================================== + +container_name(Db) -> + Suffix = list_to_binary(mem3:shard_suffix(Db)), + DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]), + couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]), + DbNameSuffix. + +hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>. + +%% ==================================================================== +%% Internal OpenStack Swift implementation +%% ==================================================================== + +extractAuthInfo([{_,_}|_] = Obj,[])-> + Storage_URL = proplists:get_value(<<"publicURL">>, Obj), + Storage_Token = proplists:get_value(<<"id">>, Obj), + Status = true, + [Storage_URL, Storage_Token, Status]; +extractAuthInfo([{[{_,_}|_]}] = Obj,[])-> + [{ObjNext}] = Obj, + Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext), + Storage_Token = proplists:get_value(<<"id">>, ObjNext), + Status = true, + [Storage_URL, Storage_Token, Status]; +extractAuthInfo([{_,_}|L]=Obj,KeyValPath)-> + [{Key, Val}|KeyValPathNext] = KeyValPath, + ObjVal = proplists:get_value(Key,Obj), + case Val of + [] -> extractAuthInfo(ObjVal, KeyValPathNext); + ObjVal -> extractAuthInfo(Obj, KeyValPathNext); + _ -> ["", "", false] + end; +extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)-> + [ObjCur| _] = Obj, + [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath), + case Status of + false -> extractAuthInfo(L, KeyValPath); + _ -> [Storage_URL, Storage_Token, Status] + end; +extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)-> + {Doc} = Obj, + extractAuthInfo(Doc, KeyValPath). + +att_processor(DbName,Att) -> + couch_log:debug("Swift: attachment processor",[]), + [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att), + [Name, Data] = couch_att:fetch([name, data], Att), + couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]), + case is_binary(Data) of + true -> + couch_log:debug("Swift: binary attachment exists",[]), + case swift_put_object(DbName, Name, Type, [], Data) of + {ok,{201,NewUrl}} -> + N1 = unicode:characters_to_binary(NewUrl, unicode, utf8), + couch_log:debug("~p ~p~n",[N1, NewUrl]), + NewAtt = couch_att:store(data,N1,Att), + couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]), + {ObjectSize,EtagMD5} = swift_head_object(DbName, Name), + NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt), + NewAtt1; + {_,{Code,_}} -> + couch_log:debug("Swift: response code is ~p~n",[Code]), + Att + end; + _ -> + Att + end. + +swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) -> + couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]), + case authenticate() of + {ok,{Url,Storage_Token}} -> + ObjNameEncoded = couch_util:url_encode(ObjName), + Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}], + Method = put, + couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]), + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url: ~p~n",[NewUrl]), + R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []), + couch_log:debug("~p~n",[R]), + case R of + {ok,_} -> + {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, + {ok,{ReturnCode,NewUrl}}; + Error -> + {error,{element(2,Error),NewUrl}} + end; + {not_authenticated,_} -> + {error,"Not authenticated",""}; + {auth_not_supported, Error} -> + {error,{element(2,Error),""}} + end. + +swift_delete_object(Container, ObjName) -> + couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container, ObjName]). + +swift_get_object(Container, ObjName) -> + couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]), + case authenticate() of + {ok,{Url,Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + ObjNameEncoded = couch_util:url_encode(ObjName), + Method = get, + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url ~p~n",[NewUrl]), + R = httpc:request(Method, {NewUrl, Header}, [], []), + {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R, + couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]), + Body; + {not_authenticated,_} -> + {error, "Not authenticated"}; + {not_supported, Error} -> + {error,{element(2,Error),""}} + end. + +swift_head_object(Container, ObjName) -> + couch_log:debug("Swift: head object ~p/~p ~n",[Container, ObjName]), + case authenticate() of + {ok,{Url,Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + Method = head, + ObjNameEncoded = couch_util:url_encode(ObjName), + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url ~p~n",[NewUrl]), + R = httpc:request(Method, {NewUrl, Header}, [], []), + {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R, + couch_log:debug("Swift: ~p~p~n",[ReturnCode, Head]), + ObjectSize = lists:filter(fun ({"content-length",_}) -> true ; (_) -> false end, Head), + EtagHeader = lists:filter(fun ({"etag",_}) -> true ; (_) -> false end, Head), + Etag = element(2,lists:nth(1,EtagHeader)), + {ObjectSizeNumeric,_} = string:to_integer(element(2,lists:nth(1,ObjectSize))), + couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",[ObjectSizeNumeric, Etag]), + couch_log:debug("Etag in base16 ~p~n",[Etag]), + EtagDecode = hex_to_bin(Etag), + EtagMD5 = base64:encode(EtagDecode), + couch_log:debug("Etag in base64 ~p~n",[EtagMD5]), + {ObjectSizeNumeric,EtagMD5}; + {auth_not_supported, Error} -> + {error,{element(2,Error),""}}; + {not_authenticated, _} -> + {-1,""} + end. + +swift_create_container(DbName) -> + couch_log:debug("Swift : create container ~p~n",[DbName]), + case authenticate() of + {ok,{Url,Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + Method = put, + Container = DbName, + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/", + couch_log:debug("Swift: url ~p ~n",[NewUrl]), + R = httpc:request(Method, {NewUrl, Header, [], []}, [], []), + case R of + {ok,_} -> + {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, + couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]), + {ok,{ReturnCode,Container}}; + Error -> + couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]), + {error,{element(2,Error),Container}} + end; + {not_authenticated,_} -> + {error, "Not authenticated"}; + {not_supported, Error} -> + {error,{element(2,Error),""}} + end. + +swift_delete_container(Container) -> + case authenticate() of + {ok,{Url,Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + Method = delete, + NewUrl = Url ++ "/" ++ Container ++"/", + R = httpc:request(Method, {NewUrl, Header, [], []}, [], []), + case R of + {ok,_} -> + {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, + {ok,{ReturnCode,Container}}; + Error -> + {error,{element(2,Error),Container}} + end; + {not_authenticated, _} -> + {error, "Not authenticated"}; + {not_supported, Error} -> + {error,{element(2,Error),""}} + end. + +authenticate() -> + couch_log:debug("Going to authenticate in Swift",[]), + case config:get("swift", "auth_model", "tempauth") of + "tempauth" -> + Method = get, + URL = config:get("swift", "auth_url"), + Header = [{"X-Storage-User", config:get("swift", "account_tenant") ++ ":" ++ config:get("swift", "username")},{"X-Storage-Pass", config:get("swift", "password")}], + couch_log:debug("Going to send authentication request: ~p~n",[URL]), + R = httpc:request(Method, {URL, Header}, [], []), + {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R, + case ReturnCode of + 200 -> + couch_log:debug("swift: authenticated: ~p~n",[ReturnCode]), + Storage_URL_F = fun ({"x-storage-url",_}) -> true ; (_) -> false end, + Storage_URL = lists:filter(Storage_URL_F, Head), + Storage_Token_F = fun ({"x-storage-token",_}) -> true ; (_) -> false end, + Storage_Token = lists:filter(Storage_Token_F, Head), + {ok,{element(2,lists:nth(1,Storage_URL)), element(2,lists:nth(1,Storage_Token))}}; + _ -> + couch_log:debug("swift: not authenticated: ~p~n",[ReturnCode]), + {not_authenticated, ""} + end; + "keystone" -> + Method = post, + URL = config:get("swift", "auth_url"), + Header = [], + Type = "application/json", + Tenant = config:get("swift", "account_tenant"), + User = config:get("swift", "username"), + Psw = config:get("swift", "password"), + Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant ++ "\", \"passwordCredentials\": {\"username\": \"" ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}", + HTTPOptions = [], + Options = [], + R = httpc:request(Method, {URL, Header, Type, Body}, HTTPOptions, Options), + {ok, {{"HTTP/1.1",ReturnCode, State}, _, Response}} = R, + case ReturnCode of + 200 -> + couch_log:debug("swift: authenticated: ~p~n",[ReturnCode]), + Json = jiffy:decode(Response), + AuthInfoBinary = extractAuthInfo(Json, [{<<"access">>, ""}, {<<"serviceCatalog">>, ""}, {<<"name">>, <<"swift">>}, {<<"endpoints">>, ""}]), + {ok, {binary_to_list(lists:nth(1,AuthInfoBinary)), binary_to_list(lists:nth(2,AuthInfoBinary))}}; + _ -> + couch_log:debug("swift: not authenticated: ~p~n",[ReturnCode]), + {not_authenticated, ""} + end; + Error -> + {auth_not_supported, Error} + end. + diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index a7f4ed9..32e6f3f 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -164,6 +164,21 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix) -> {[[<<"add">>, Range, Node] | Raw], orddict:append(Node, Range, ByNode), orddict:append(Range, Node, ByRange)} end, {[], [], []}, Shards), + + case fabric_attachments_handler:externalize_att(DbName) of + "true" -> + couch_log:debug("Going to create container ~p, ~p~n",[DbName, Suffix]), + DbNameSuffix = unicode:characters_to_list(DbName) ++ Suffix, + couch_log:debug("New name is ~p~n",[DbNameSuffix]), + case fabric_attachments_handler:container_handler(create,DbNameSuffix) of + {ok, Container} -> + couch_log:debug("Container created",[]); + {error,_} -> + couch_log:debug("Container creation failed",[]) + end; + "false" -> + couch_log:debug("Store attachmets externally is disabled",[]) + end, #doc{id=DbName, body = {[ {<<"shard_suffix">>, Suffix}, {<<"changelog">>, lists:sort(RawOut)}, From 220150ae5928cf71d1be5f33ebe2e3140d5d82a1 Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Mon, 4 Jan 2016 10:20:48 +0200 Subject: [PATCH 2/3] improved code. based on the previous review --- src/fabric_att_handler.erl | 136 ++++++++++++ src/fabric_attachments_handler.erl | 333 ----------------------------- src/fabric_db_create.erl | 14 +- src/fabric_swift_driver.erl | 269 +++++++++++++++++++++++ 4 files changed, 411 insertions(+), 341 deletions(-) create mode 100644 src/fabric_att_handler.erl delete mode 100644 src/fabric_attachments_handler.erl create mode 100644 src/fabric_swift_driver.erl diff --git a/src/fabric_att_handler.erl b/src/fabric_att_handler.erl new file mode 100644 index 0000000..ce1aa45 --- /dev/null +++ b/src/fabric_att_handler.erl @@ -0,0 +1,136 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(fabric_att_handler). + +-include_lib("fabric/include/fabric.hrl"). +-include_lib("couch/include/couch_db.hrl"). + +-export([att_store/2, att/3, container/2, att_store/5, external_store/0]). + + +%% ==================================================================== +%% External API functions. Exposes a generic API that can be used with +%% other backend drivers +%% ==================================================================== + +att_store(FileName, Db, ContentLen, MimeType, Req) -> + ContainerName = container_name(Db), + couch_log:debug("Going to store ~p of length is ~p," + ++ " ~p in the container: ~n",[FileName, ContentLen, + ContainerName]), + %TO-DO: No chunk reader. All kept in the memory. Should be fixed. + Data = couch_httpd:recv(Req, ContentLen), + case (get_backend_driver()):put_object(ContainerName, FileName, MimeType, + [], Data) of + {ok,{"201",NewUrl}} -> + couch_log:debug("Object ~p created.Response code is 201 ~n", + [FileName]), + case (get_backend_driver()):head_object(ContainerName, FileName) of + {ok, {ObjectSize, EtagMD5}} -> + {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8), + ObjectSize, EtagMD5, (get_backend_driver()):store_id()}}; + {error, _} -> + {error, Data} + end; + {error, _} -> + {error, Data} + end. + +att_store(Db, #doc{}=Doc) -> + att_store(Db, [Doc]); +att_store(Db, Docs) when is_list(Docs) -> + DbName = container_name(Db), + [#doc{atts=Atts0} = Doc | Rest] = Docs, + if + length(Atts0) > 0 -> + Atts = [att_processor(DbName,Att) || Att <- Atts0], + [Doc#doc{atts = Atts} | Rest]; + true -> + Docs + end. + +att(get, Db, Att) -> + DbName = container_name(Db), + [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len, + att_external_size], Att), + couch_log:debug("Going to retrieve ~p. DB length is: ~p, " + ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]), + NewData = (get_backend_driver()):get_object(DbName, Name), + NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser}, + {disk_len, AttLenUser}], Att), + NewAtt; +att(delete,Db, Att) -> + DbName = container_name(Db), + [Name] = couch_att:fetch([name],Att), + couch_log:debug("Delete ~p from ~p", [Name, DbName]). + +container(create,DbName) -> + couch_log:debug("Create container ~p~n", [DbName]), + case (get_backend_driver()):create_container(DbName) of + {ok, {"201", Container}} -> + couch_log:debug("Container ~p created succesfully ~n", [Container]), + {ok, Container}; + {error, _} -> + couch_log:debug("Container ~p creation failed ~n", [DbName]), + {error, DbName} + end; +container(get, DbName) -> + couch_log:debug("Get container ~p~n", [DbName]); +container(delete, DbName)-> + couch_log:debug("Delete container ~p~n", [DbName]), + (get_backend_driver()):delete_container(DbName). + +external_store() -> + config:get_boolean("ext_store", "attachments_offload", false). + +%% ==================================================================== +%% Internal general functions +%% ==================================================================== + +get_backend_driver() -> + case config:get("ext_store", "active_store") of + "swift" -> + fabric_swift_driver; + undefined -> + fabric_swift_driver + end. + +container_name(Db) -> + Suffix = list_to_binary(mem3:shard_suffix(Db)), + DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]), + DbNameSuffix. + +att_processor(DbName, Att) -> + [Name, Data, Type, Enc] = couch_att:fetch([name, data, type, encoding], Att), + couch_log:debug("Att name: ~p, Type: ~p, Encoding: ~p ~n", [Name, Type, Enc]), + case is_binary(Data) of + true -> + case (get_backend_driver()):put_object(DbName, Name, Type, [], Data) of + {ok, {"201", NewUrl}} -> + N1 = unicode:characters_to_binary(NewUrl, unicode, utf8), + NewAtt = couch_att:store(data, N1, Att), + case (get_backend_driver()):head_object(DbName, Name) of + {ok, {ObjectSize,EtagMD5}} -> + NewAtt1 = couch_att:store([{att_extstore_size, ObjectSize}, + {att_extstore_id, (get_backend_driver()):store_id()}, + {att_extstore_md5, EtagMD5}], NewAtt), + NewAtt1; + {error, _} -> + Att + end; + {error, _} -> + Att + end; + _ -> + Att + end. diff --git a/src/fabric_attachments_handler.erl b/src/fabric_attachments_handler.erl deleted file mode 100644 index c4aab32..0000000 --- a/src/fabric_attachments_handler.erl +++ /dev/null @@ -1,333 +0,0 @@ -%% @author gilv -%% @doc @todo Add description to fabric_swift_handler. - - --module(fabric_attachments_handler). - --include_lib("fabric/include/fabric.hrl"). --include_lib("couch/include/couch_db.hrl"). - --export([inline_att_store/2, inline_att_handler/3, container_handler/2, normal_att_store/5, externalize_att/1]). - - -%% ==================================================================== -%% External API functions. I try to keep them as general as possible, -%% without correlation to specific object store that might be internally used. -%% ==================================================================== - -normal_att_store(FileName,Db,ContentLen,MimeType,Req) -> - ContainerName = container_name(Db), - couch_log:debug("Standard attachment handler",[]), - couch_log:debug("Going to store ~p of length is ~p, ~p in the container: ~n",[FileName,ContentLen,ContainerName]), - %Bad implementation - no chunk reader. All kept in the memory. Should be fixed. - Data = couch_httpd:recv(Req, ContentLen), - case swift_put_object(ContainerName, FileName, MimeType, [], Data) of - {ok,{201,NewUrl}} -> - couch_log:debug("Created. Swift response code is 201 ~n",[]), - {ObjectSize,EtagMD5} = swift_head_object(ContainerName, FileName), - {unicode:characters_to_binary(NewUrl, unicode, utf8), ObjectSize, EtagMD5}; - {_,{Code,_}} -> - couch_log:debug("Swift response code is ~p~n",[]), - {Data, -1} - end. - -inline_att_store(Db, Docs) -> - DbName = container_name(Db), - couch_log:debug("Store inline base64 encoded attachment ~n",[]), - if - is_list(Docs) -> - couch_log:debug("Going to handle document list",[]), - DocsArray = Docs; - true -> - couch_log:debug("Going to handle single document",[]), - DocsArray = [Docs] - end, - [#doc{atts=Atts0} = Doc | Rest] = DocsArray, - if - length(Atts0) > 0 -> - couch_log:debug("Att length is larger than 0",[]), - Atts = [att_processor(DbName,Att) || Att <- Atts0], - if - is_list(Docs) -> - [Doc#doc{atts = Atts} | Rest]; - true -> - Doc#doc{atts = Atts} - end; - true -> - couch_log:debug("No attachments to handle",[]), - Docs - end. - -inline_att_handler(get, Db, Att) -> - DbName = container_name(Db), - couch_log:debug("Retrieve attachment",[]), - [Data,Name,AttLen,AttLenUser] = couch_att:fetch([data, name,att_len, att_external_size],Att), - couch_log:debug("Going to retrieve ~p. DB length is: ~p, stored length: ~p~n",[Name, AttLen, AttLenUser]), - NewData = swift_get_object(DbName, Name), - NewAtt = couch_att:store([{data, NewData},{att_len,AttLenUser},{disk_len,AttLenUser}], Att), - NewAtt; - -inline_att_handler(delete,Db, Att) -> - DbName = container_name(Db), - couch_log:debug("Delete attachment ~p~n",[]). - -container_handler(create,DbName) -> - couch_log:debug("Create container ~p~n",[DbName]), - case swift_create_container(DbName) of - {ok,{201,Container}} -> - couch_log:debug("Container ~p created succesfully ~n",[Container]), - {ok,Container}; - {error,_} -> - couch_log:debug("Container ~p creation failed ~n",[DbName]), - {error,DbName} - end; -container_handler(get,DbName) -> - couch_log:debug("Get container ~p~n",[DbName]); -container_handler(delete,DbName)-> - couch_log:debug("Delete container ~p~n",[DbName]), - swift_delete_container(DbName). - -externalize_att(Db) -> - Res = config:get("swift","attachments_offload","false"), - Res. - -%% ==================================================================== -%% Internal general functions -%% ==================================================================== - -container_name(Db) -> - Suffix = list_to_binary(mem3:shard_suffix(Db)), - DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]), - couch_log:debug("Db to Container name ~p~n",[DbNameSuffix]), - DbNameSuffix. - -hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>. - -%% ==================================================================== -%% Internal OpenStack Swift implementation -%% ==================================================================== - -extractAuthInfo([{_,_}|_] = Obj,[])-> - Storage_URL = proplists:get_value(<<"publicURL">>, Obj), - Storage_Token = proplists:get_value(<<"id">>, Obj), - Status = true, - [Storage_URL, Storage_Token, Status]; -extractAuthInfo([{[{_,_}|_]}] = Obj,[])-> - [{ObjNext}] = Obj, - Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext), - Storage_Token = proplists:get_value(<<"id">>, ObjNext), - Status = true, - [Storage_URL, Storage_Token, Status]; -extractAuthInfo([{_,_}|L]=Obj,KeyValPath)-> - [{Key, Val}|KeyValPathNext] = KeyValPath, - ObjVal = proplists:get_value(Key,Obj), - case Val of - [] -> extractAuthInfo(ObjVal, KeyValPathNext); - ObjVal -> extractAuthInfo(Obj, KeyValPathNext); - _ -> ["", "", false] - end; -extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)-> - [ObjCur| _] = Obj, - [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath), - case Status of - false -> extractAuthInfo(L, KeyValPath); - _ -> [Storage_URL, Storage_Token, Status] - end; -extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)-> - {Doc} = Obj, - extractAuthInfo(Doc, KeyValPath). - -att_processor(DbName,Att) -> - couch_log:debug("Swift: attachment processor",[]), - [Type, Enc, DiskLen, AttLen] = couch_att:fetch([type, encoding, disk_len, att_len], Att), - [Name, Data] = couch_att:fetch([name, data], Att), - couch_log:debug("Swift: att name: ~p, type: ~p, encoding: ~p, disk len: ~p~n",[Name,Type,Enc,DiskLen]), - case is_binary(Data) of - true -> - couch_log:debug("Swift: binary attachment exists",[]), - case swift_put_object(DbName, Name, Type, [], Data) of - {ok,{201,NewUrl}} -> - N1 = unicode:characters_to_binary(NewUrl, unicode, utf8), - couch_log:debug("~p ~p~n",[N1, NewUrl]), - NewAtt = couch_att:store(data,N1,Att), - couch_log:debug("Swift. testing store in original length ~p~n",[AttLen]), - {ObjectSize,EtagMD5} = swift_head_object(DbName, Name), - NewAtt1 = couch_att:store([{att_external_size,ObjectSize},{att_external,"external"},{att_external_md5,EtagMD5}],NewAtt), - NewAtt1; - {_,{Code,_}} -> - couch_log:debug("Swift: response code is ~p~n",[Code]), - Att - end; - _ -> - Att - end. - -swift_put_object(Container, ObjName, ContextType, CustomHeaders, Data) -> - couch_log:debug("Swift: PUT ~p/~s object method ~n",[Container, ObjName]), - case authenticate() of - {ok,{Url,Storage_Token}} -> - ObjNameEncoded = couch_util:url_encode(ObjName), - Headers = CustomHeaders ++ [{"X-Auth-Token",Storage_Token}], - Method = put, - couch_log:debug("Swift: going to upload : ~p ~p ~p ~p ~p~n",[Url, Headers, ContextType,ObjNameEncoded, Container]), - NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url: ~p~n",[NewUrl]), - R = httpc:request(Method, {NewUrl, Headers, unicode:characters_to_list(ContextType), Data}, [], []), - couch_log:debug("~p~n",[R]), - case R of - {ok,_} -> - {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, - {ok,{ReturnCode,NewUrl}}; - Error -> - {error,{element(2,Error),NewUrl}} - end; - {not_authenticated,_} -> - {error,"Not authenticated",""}; - {auth_not_supported, Error} -> - {error,{element(2,Error),""}} - end. - -swift_delete_object(Container, ObjName) -> - couch_log:debug("Swift: Delete ~p/~p object method ~n",[Container, ObjName]). - -swift_get_object(Container, ObjName) -> - couch_log:debug("Swift: get object ~p/~p ~n",[Container, ObjName]), - case authenticate() of - {ok,{Url,Storage_Token}} -> - Header = [{"X-Auth-Token",Storage_Token}], - ObjNameEncoded = couch_util:url_encode(ObjName), - Method = get, - NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url ~p~n",[NewUrl]), - R = httpc:request(Method, {NewUrl, Header}, [], []), - {ok, {{"HTTP/1.1",ReturnCode, _}, Head, Body}} = R, - couch_log:debug("Swift: ~p~p ~n",[ReturnCode, Head]), - Body; - {not_authenticated,_} -> - {error, "Not authenticated"}; - {not_supported, Error} -> - {error,{element(2,Error),""}} - end. - -swift_head_object(Container, ObjName) -> - couch_log:debug("Swift: head object ~p/~p ~n",[Container, ObjName]), - case authenticate() of - {ok,{Url,Storage_Token}} -> - Header = [{"X-Auth-Token",Storage_Token}], - Method = head, - ObjNameEncoded = couch_util:url_encode(ObjName), - NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url ~p~n",[NewUrl]), - R = httpc:request(Method, {NewUrl, Header}, [], []), - {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R, - couch_log:debug("Swift: ~p~p~n",[ReturnCode, Head]), - ObjectSize = lists:filter(fun ({"content-length",_}) -> true ; (_) -> false end, Head), - EtagHeader = lists:filter(fun ({"etag",_}) -> true ; (_) -> false end, Head), - Etag = element(2,lists:nth(1,EtagHeader)), - {ObjectSizeNumeric,_} = string:to_integer(element(2,lists:nth(1,ObjectSize))), - couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n",[ObjectSizeNumeric, Etag]), - couch_log:debug("Etag in base16 ~p~n",[Etag]), - EtagDecode = hex_to_bin(Etag), - EtagMD5 = base64:encode(EtagDecode), - couch_log:debug("Etag in base64 ~p~n",[EtagMD5]), - {ObjectSizeNumeric,EtagMD5}; - {auth_not_supported, Error} -> - {error,{element(2,Error),""}}; - {not_authenticated, _} -> - {-1,""} - end. - -swift_create_container(DbName) -> - couch_log:debug("Swift : create container ~p~n",[DbName]), - case authenticate() of - {ok,{Url,Storage_Token}} -> - Header = [{"X-Auth-Token",Storage_Token}], - Method = put, - Container = DbName, - NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/", - couch_log:debug("Swift: url ~p ~n",[NewUrl]), - R = httpc:request(Method, {NewUrl, Header, [], []}, [], []), - case R of - {ok,_} -> - {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, - couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]), - {ok,{ReturnCode,Container}}; - Error -> - couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]), - {error,{element(2,Error),Container}} - end; - {not_authenticated,_} -> - {error, "Not authenticated"}; - {not_supported, Error} -> - {error,{element(2,Error),""}} - end. - -swift_delete_container(Container) -> - case authenticate() of - {ok,{Url,Storage_Token}} -> - Header = [{"X-Auth-Token",Storage_Token}], - Method = delete, - NewUrl = Url ++ "/" ++ Container ++"/", - R = httpc:request(Method, {NewUrl, Header, [], []}, [], []), - case R of - {ok,_} -> - {ok, {{"HTTP/1.1",ReturnCode, _}, _, _}} = R, - {ok,{ReturnCode,Container}}; - Error -> - {error,{element(2,Error),Container}} - end; - {not_authenticated, _} -> - {error, "Not authenticated"}; - {not_supported, Error} -> - {error,{element(2,Error),""}} - end. - -authenticate() -> - couch_log:debug("Going to authenticate in Swift",[]), - case config:get("swift", "auth_model", "tempauth") of - "tempauth" -> - Method = get, - URL = config:get("swift", "auth_url"), - Header = [{"X-Storage-User", config:get("swift", "account_tenant") ++ ":" ++ config:get("swift", "username")},{"X-Storage-Pass", config:get("swift", "password")}], - couch_log:debug("Going to send authentication request: ~p~n",[URL]), - R = httpc:request(Method, {URL, Header}, [], []), - {ok, {{"HTTP/1.1",ReturnCode, _}, Head, _}} = R, - case ReturnCode of - 200 -> - couch_log:debug("swift: authenticated: ~p~n",[ReturnCode]), - Storage_URL_F = fun ({"x-storage-url",_}) -> true ; (_) -> false end, - Storage_URL = lists:filter(Storage_URL_F, Head), - Storage_Token_F = fun ({"x-storage-token",_}) -> true ; (_) -> false end, - Storage_Token = lists:filter(Storage_Token_F, Head), - {ok,{element(2,lists:nth(1,Storage_URL)), element(2,lists:nth(1,Storage_Token))}}; - _ -> - couch_log:debug("swift: not authenticated: ~p~n",[ReturnCode]), - {not_authenticated, ""} - end; - "keystone" -> - Method = post, - URL = config:get("swift", "auth_url"), - Header = [], - Type = "application/json", - Tenant = config:get("swift", "account_tenant"), - User = config:get("swift", "username"), - Psw = config:get("swift", "password"), - Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant ++ "\", \"passwordCredentials\": {\"username\": \"" ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}", - HTTPOptions = [], - Options = [], - R = httpc:request(Method, {URL, Header, Type, Body}, HTTPOptions, Options), - {ok, {{"HTTP/1.1",ReturnCode, State}, _, Response}} = R, - case ReturnCode of - 200 -> - couch_log:debug("swift: authenticated: ~p~n",[ReturnCode]), - Json = jiffy:decode(Response), - AuthInfoBinary = extractAuthInfo(Json, [{<<"access">>, ""}, {<<"serviceCatalog">>, ""}, {<<"name">>, <<"swift">>}, {<<"endpoints">>, ""}]), - {ok, {binary_to_list(lists:nth(1,AuthInfoBinary)), binary_to_list(lists:nth(2,AuthInfoBinary))}}; - _ -> - couch_log:debug("swift: not authenticated: ~p~n",[ReturnCode]), - {not_authenticated, ""} - end; - Error -> - {auth_not_supported, Error} - end. - diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index 32e6f3f..543ec0e 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -165,18 +165,16 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix) -> orddict:append(Range, Node, ByRange)} end, {[], [], []}, Shards), - case fabric_attachments_handler:externalize_att(DbName) of - "true" -> - couch_log:debug("Going to create container ~p, ~p~n",[DbName, Suffix]), + case fabric_att_handler:external_store() of + true -> DbNameSuffix = unicode:characters_to_list(DbName) ++ Suffix, - couch_log:debug("New name is ~p~n",[DbNameSuffix]), - case fabric_attachments_handler:container_handler(create,DbNameSuffix) of + case fabric_att_handler:container(create,DbNameSuffix) of {ok, Container} -> - couch_log:debug("Container created",[]); + couch_log:debug("Container ~p created", [Container]); {error,_} -> - couch_log:debug("Container creation failed",[]) + couch_log:debug("Container ~p creation failed", [DbNameSuffix]) end; - "false" -> + false -> couch_log:debug("Store attachmets externally is disabled",[]) end, #doc{id=DbName, body = {[ diff --git a/src/fabric_swift_driver.erl b/src/fabric_swift_driver.erl new file mode 100644 index 0000000..ff2e0eb --- /dev/null +++ b/src/fabric_swift_driver.erl @@ -0,0 +1,269 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(fabric_swift_driver). + +-export([put_object/5, delete_object/2, get_object/2, head_object/2]). +-export([store_id/0, create_container/1, delete_container/1 ]). +%% ==================================================================== +%% OpenStack Swift implementation +%% ==================================================================== + +store_id() -> + "swift". + +put_object(Container, ObjName, ContextType, CustomHeaders, Data) -> + couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]), + case authenticate() of + {ok, {Url, Storage_Token}} -> + ObjNameEncoded = couch_util:url_encode(ObjName), + Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token}, + {"Content-Type", + unicode:characters_to_list(ContextType)}], + Method = put, + couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n", + [Url,ContextType, ObjNameEncoded, Container]), + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]), + R = ibrowse:send_req(NewUrl, Headers, Method, Data), + case R of + {ok, ReturnCode, _, _} -> + {ok, {ReturnCode, NewUrl}}; + Error -> + couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]), + {error, "PUT object failed"} + end; + {not_authenticated, _} -> + {error, "Not authenticated"} + end. + +delete_object(Container, ObjName) -> + %TO-DO: should be called during database compaction. + couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]), + case authenticate() of + {ok, {Url, Storage_Token}} -> + Header = [{"X-Auth-Token", Storage_Token}], + ObjNameEncoded = couch_util:url_encode(ObjName), + Method = delete, + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" + ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url ~p~n", [NewUrl]), + R = ibrowse:send_req(NewUrl, Header, Method), + case R of + {ok, ReturnCode, _, _} -> + couch_log:debug("Swift: Delete ~p. Return code ~p ~n", + [NewUrl, ReturnCode]), + {ok, ReturnCode}; + Error -> + couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]), + couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + {error, "Delete object failed"} + end; + {not_authenticated, _} -> + {error, "Not authenticated"} + end. + +get_object(Container, ObjName) -> + couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]), + case authenticate() of + {ok, {Url, Storage_Token}} -> + Header = [{"X-Auth-Token", Storage_Token}], + ObjNameEncoded = couch_util:url_encode(ObjName), + Method = get, + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" + ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url ~p~n", [NewUrl]), + R = ibrowse:send_req(NewUrl, Header, Method), + case R of + {ok, ReturnCode, _, Body} -> + couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]), + Body; + Error -> + couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]), + couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + {error, "Get object failed"} + end; + {not_authenticated, _} -> + {error, "Not authenticated"} + end. + +head_object(Container, ObjName) -> + couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]), + case authenticate() of + {ok, {Url, Storage_Token}} -> + Header = [{"X-Auth-Token", Storage_Token}], + Method = head, + ObjNameEncoded = couch_util:url_encode(ObjName), + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) + ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), + couch_log:debug("Swift: url ~p~n", [NewUrl]), + R = ibrowse:send_req(NewUrl, Header, Method), + case R of + {ok, ReturnCode, Head, _} -> + couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]), + ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head), + EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head), + Etag = element(2, lists:nth(1, EtagHeader)), + {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))), + couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n", + [ObjectSizeNumeric, Etag]), + EtagDecode = hex_to_bin(Etag), + EtagMD5 = base64:encode(EtagDecode), + couch_log:debug("Etag in base64 ~p~n", [EtagMD5]), + {ok, {ObjectSizeNumeric, EtagMD5}}; + Error -> + couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]), + couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + {error,"HEAD object failed"} + end; + {not_authenticated, _} -> + {error, "Not authenticated"} + end. + +create_container(DbName) -> + couch_log:debug("Swift : create container ~p~n", [DbName]), + case authenticate() of + {ok, {Url, Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + Method = put, + Container = DbName, + NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/", + couch_log:debug("Swift: url ~p ~n",[NewUrl]), + R = ibrowse:send_req(NewUrl, Header, Method), + case R of + {ok, ReturnCode, _, _} -> + couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]), + {ok,{ReturnCode,Container}}; + Error -> + couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]), + {error, "Failed to create container"} + end; + {not_authenticated,_} -> + {error, "Failed to create container. Not authenticated"} + end. + +delete_container(Container) -> + case authenticate() of + {ok,{Url,Storage_Token}} -> + Header = [{"X-Auth-Token",Storage_Token}], + Method = delete, + NewUrl = Url ++ "/" ++ Container ++"/", + R = ibrowse:send_req(NewUrl, Header, Method), + case R of + {ok, ReturnCode, _, _} -> + {ok,{ReturnCode,Container}}; + Error -> + {error,{element(2,Error),Container}} + end; + {not_authenticated, _} -> + {error, "Not authenticated"}; + {not_supported, Error} -> + {error,{element(2,Error),""}} + end. + +%% ==================================================================== +%% Internal functions +%% ==================================================================== + +hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>. + +authenticate() -> + couch_log:debug("Going to authenticate in Swift",[]), + case config:get("swift", "auth_model", "tempauth") of + "tempauth" -> + swift_v1_auth(); + "keystone" -> + keystone_auth(); + Error -> + couch_log:debug("Authentication method is not supported: ~p", element(2,Error)), + {not_authenticated, ""} + end. + +keystone_auth() -> + Method = post, + URL = config:get("swift", "auth_url"), + Header = [{"Content-Type", "application/json"}], + Tenant = config:get("swift", "account_tenant"), + User = config:get("swift", "username"), + Psw = config:get("swift", "password"), + Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant + ++ "\", \"passwordCredentials\": {\"username\": \"" + ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}", + R = ibrowse:send_req(URL, Header, Method, Body), + {ok, ReturnCode, _, Response} = R, + case ReturnCode of + "200" -> + couch_log:debug("keystone: authenticated: ~p~n", [ReturnCode]), + Json = jiffy:decode(Response), + AuthInfoBinary = extractAuthInfo(Json, [{<<"access">>, ""}, + {<<"serviceCatalog">>, ""}, + {<<"name">>, <<"swift">>}, + {<<"endpoints">>, ""}]), + {ok, {binary_to_list(lists:nth(1, AuthInfoBinary)), + binary_to_list(lists:nth(2, AuthInfoBinary))}}; + _ -> + couch_log:debug("keystone: not authenticated: ~p~n", [ReturnCode]), + {not_authenticated, ""} + end. + +swift_v1_auth() -> + Method = get, + URL = config:get("swift", "auth_url"), + Header = [{"X-Storage-User", config:get("swift", "account_tenant") ++ ":" + ++ config:get("swift", "username")}, + {"X-Storage-Pass", config:get("swift", "password")}], + R = ibrowse:send_req(URL, Header, Method), + {ok, ReturnCode, Head, _} = R, + case ReturnCode of + "200" -> + couch_log:debug("swift v1: authenticated: ~p~n", [ReturnCode]), + Storage_URL_F = fun ({"X-Storage-Url", _}) -> true ; (_) -> false end, + Storage_URL = lists:filter(Storage_URL_F, Head), + Storage_Token_F = fun ({"X-Storage-Token", _}) -> true ; (_) -> false end, + Storage_Token = lists:filter(Storage_Token_F, Head), + {ok,{element(2,lists:nth(1, Storage_URL)), element(2,lists:nth(1, Storage_Token))}}; + _ -> + couch_log:debug("swift v1: not authenticated: ~p~n", [ReturnCode]), + {not_authenticated, ReturnCode} + end. + +extractAuthInfo([{_,_}|_] = Obj,[])-> + Storage_URL = proplists:get_value(<<"publicURL">>, Obj), + Storage_Token = proplists:get_value(<<"id">>, Obj), + Status = true, + [Storage_URL, Storage_Token, Status]; +extractAuthInfo([{[{_,_}|_]}] = Obj, [])-> + [{ObjNext}] = Obj, + Storage_URL = proplists:get_value(<<"publicURL">>, ObjNext), + Storage_Token = proplists:get_value(<<"id">>, ObjNext), + Status = true, + [Storage_URL, Storage_Token, Status]; +extractAuthInfo([{_,_}|_]=Obj,KeyValPath)-> + [{Key, Val}|KeyValPathNext] = KeyValPath, + ObjVal = proplists:get_value(Key,Obj), + case Val of + [] -> extractAuthInfo(ObjVal, KeyValPathNext); + ObjVal -> extractAuthInfo(Obj, KeyValPathNext); + _ -> ["", "", false] + end; +extractAuthInfo([{[{_,_}|_]}|L]=Obj,KeyValPath)-> + [ObjCur| _] = Obj, + [Storage_URL, Storage_Token, Status] = extractAuthInfo(ObjCur, KeyValPath), + case Status of + false -> extractAuthInfo(L, KeyValPath); + _ -> [Storage_URL, Storage_Token, Status] + end; +extractAuthInfo(Obj,KeyValPath) when is_tuple(Obj)-> + {Doc} = Obj, + extractAuthInfo(Doc, KeyValPath). + + From a8fb113757c508c4ca886c03ab21c214a44c14c2 Mon Sep 17 00:00:00 2001 From: Gil Vernik Date: Wed, 20 Jan 2016 09:30:19 +0200 Subject: [PATCH 3/3] code modification based on the recent reviews --- src/fabric_att_handler.erl | 134 ++++++++++++++++++++++-------------- src/fabric_db_create.erl | 2 +- src/fabric_swift_driver.erl | 131 +++++++++++++++++++++-------------- 3 files changed, 161 insertions(+), 106 deletions(-) diff --git a/src/fabric_att_handler.erl b/src/fabric_att_handler.erl index ce1aa45..caf164c 100644 --- a/src/fabric_att_handler.erl +++ b/src/fabric_att_handler.erl @@ -15,7 +15,7 @@ -include_lib("fabric/include/fabric.hrl"). -include_lib("couch/include/couch_db.hrl"). --export([att_store/2, att/3, container/2, att_store/5, external_store/0]). +-export([att_store/2, att_get/2, att_delete/2, dataroot/2, att_store/5, external_store/0]). %% ==================================================================== @@ -24,71 +24,97 @@ %% ==================================================================== att_store(FileName, Db, ContentLen, MimeType, Req) -> - ContainerName = container_name(Db), - couch_log:debug("Going to store ~p of length is ~p," - ++ " ~p in the container: ~n",[FileName, ContentLen, - ContainerName]), + DatarootName = dataroot_name(Db), + couch_log:info("Going to store ~p of length is ~p," + ++ " in the dataroot: ~p ~n",[FileName, ContentLen, + DatarootName]), %TO-DO: No chunk reader. All kept in the memory. Should be fixed. - Data = couch_httpd:recv(Req, ContentLen), - case (get_backend_driver()):put_object(ContainerName, FileName, MimeType, - [], Data) of - {ok,{"201",NewUrl}} -> - couch_log:debug("Object ~p created.Response code is 201 ~n", - [FileName]), - case (get_backend_driver()):head_object(ContainerName, FileName) of - {ok, {ObjectSize, EtagMD5}} -> - {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8), - ObjectSize, EtagMD5, (get_backend_driver()):store_id()}}; + Data = recv(Req, ContentLen), + case get_backend_driver() of + {ok, BackendDriver} -> + Headers = [{"Content-Length", ContentLen}], + case (BackendDriver):put_object(DatarootName, FileName, MimeType, + Headers, Data) of + {ok, NewUrl} -> + case (BackendDriver):head_object(DatarootName, FileName) of + {ok, {ObjectSize, EtagMD5}} -> + {ok, {unicode:characters_to_binary(NewUrl, unicode, utf8), + ObjectSize, EtagMD5, (BackendDriver):store_id()}}; + {error, _} -> + {error, Data} + end; {error, _} -> {error, Data} end; - {error, _} -> + {error, undefined} -> {error, Data} end. att_store(Db, #doc{}=Doc) -> att_store(Db, [Doc]); att_store(Db, Docs) when is_list(Docs) -> - DbName = container_name(Db), + DatarootName = dataroot_name(Db), [#doc{atts=Atts0} = Doc | Rest] = Docs, if length(Atts0) > 0 -> - Atts = [att_processor(DbName,Att) || Att <- Atts0], + Atts = [att_processor(DatarootName,Att) || Att <- Atts0], [Doc#doc{atts = Atts} | Rest]; true -> Docs end. -att(get, Db, Att) -> - DbName = container_name(Db), +att_get(Db, Att) -> + DatarootName = dataroot_name(Db), [Name,AttLen,AttLenUser] = couch_att:fetch([name, att_len, - att_external_size], Att), - couch_log:debug("Going to retrieve ~p. DB length is: ~p, " + att_extstore_size], Att), + couch_log:info("Going to retrieve ~p. DB length is: ~p, " ++ "stored length: ~p~n", [Name, AttLen, AttLenUser]), - NewData = (get_backend_driver()):get_object(DbName, Name), - NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser}, - {disk_len, AttLenUser}], Att), - NewAtt; -att(delete,Db, Att) -> - DbName = container_name(Db), + case get_backend_driver() of + {ok, BackendDriver} -> + case (BackendDriver):get_object(DatarootName, Name) of + {ok, NewData} -> + NewAtt = couch_att:store([{data, NewData}, {att_len, AttLenUser}, + {disk_len, AttLenUser}], Att), + {ok, NewAtt}; + {error, _} -> + {error, "Get att ~p~n failed",[Name]} + end; + {error, undefined} -> + {error, "Get att ~p~n failed",[Name]} + end. + +att_delete(Db, Att) -> + %% Will be called during database compaction. + %% To-Do. + DatarootName = dataroot_name(Db), [Name] = couch_att:fetch([name],Att), - couch_log:debug("Delete ~p from ~p", [Name, DbName]). - -container(create,DbName) -> - couch_log:debug("Create container ~p~n", [DbName]), - case (get_backend_driver()):create_container(DbName) of - {ok, {"201", Container}} -> - couch_log:debug("Container ~p created succesfully ~n", [Container]), - {ok, Container}; - {error, _} -> - couch_log:debug("Container ~p creation failed ~n", [DbName]), - {error, DbName} + couch_log:debug("Delete ~p from ~p", [Name, DatarootName]). + +dataroot(create, DbName) -> + couch_log:info("Create dataroot ~p~n", [DbName]), + case get_backend_driver() of + {ok, BackendDriver} -> + case (BackendDriver):create_dataroot(DbName) of + {ok, Dataroot} -> + couch_log:debug("Dataroot ~p created succesfully ~n", [Dataroot]), + {ok, Dataroot}; + {error, _} -> + couch_log:debug("Dataroot ~p creation failed ~n", [DbName]), + {error, DbName} + end; + {error, undefined} -> + {error, "Create ~p~n failed",[DbName]} end; -container(get, DbName) -> - couch_log:debug("Get container ~p~n", [DbName]); -container(delete, DbName)-> - couch_log:debug("Delete container ~p~n", [DbName]), - (get_backend_driver()):delete_container(DbName). +dataroot(delete, DbName)-> + %% Will be called when database is completely deleted. + %% To-Do + couch_log:debug("Delete dataroot ~p~n", [DbName]), + case get_backend_driver() of + {ok, BackendDriver} -> + (BackendDriver):delete_dataroot(DbName); + {error, undefined} -> + {error, "Delete ~p~n failed",[DbName]} + end. external_store() -> config:get_boolean("ext_store", "attachments_offload", false). @@ -97,32 +123,36 @@ external_store() -> %% Internal general functions %% ==================================================================== +recv(#httpd{mochi_req=MochiReq}, Len) -> + MochiReq:recv(Len). + get_backend_driver() -> - case config:get("ext_store", "active_store") of + case config:get("ext_store", "active_store", undefined) of "swift" -> - fabric_swift_driver; + {ok, fabric_swift_driver }; undefined -> - fabric_swift_driver + {error, undefined} end. -container_name(Db) -> +dataroot_name(Db) -> Suffix = list_to_binary(mem3:shard_suffix(Db)), DbNameSuffix = erlang:iolist_to_binary([fabric:dbname(Db), Suffix]), DbNameSuffix. att_processor(DbName, Att) -> + BackendDriver = get_backend_driver(), [Name, Data, Type, Enc] = couch_att:fetch([name, data, type, encoding], Att), couch_log:debug("Att name: ~p, Type: ~p, Encoding: ~p ~n", [Name, Type, Enc]), case is_binary(Data) of true -> - case (get_backend_driver()):put_object(DbName, Name, Type, [], Data) of - {ok, {"201", NewUrl}} -> + case (BackendDriver):put_object(DbName, Name, Type, [], Data) of + {ok, NewUrl} -> N1 = unicode:characters_to_binary(NewUrl, unicode, utf8), NewAtt = couch_att:store(data, N1, Att), - case (get_backend_driver()):head_object(DbName, Name) of + case (BackendDriver):head_object(DbName, Name) of {ok, {ObjectSize,EtagMD5}} -> NewAtt1 = couch_att:store([{att_extstore_size, ObjectSize}, - {att_extstore_id, (get_backend_driver()):store_id()}, + {att_extstore_id, (BackendDriver):store_id()}, {att_extstore_md5, EtagMD5}], NewAtt), NewAtt1; {error, _} -> diff --git a/src/fabric_db_create.erl b/src/fabric_db_create.erl index 543ec0e..9146d6d 100644 --- a/src/fabric_db_create.erl +++ b/src/fabric_db_create.erl @@ -168,7 +168,7 @@ make_document([#shard{dbname=DbName}|_] = Shards, Suffix) -> case fabric_att_handler:external_store() of true -> DbNameSuffix = unicode:characters_to_list(DbName) ++ Suffix, - case fabric_att_handler:container(create,DbNameSuffix) of + case fabric_att_handler:dataroot(create, DbNameSuffix) of {ok, Container} -> couch_log:debug("Container ~p created", [Container]); {error,_} -> diff --git a/src/fabric_swift_driver.erl b/src/fabric_swift_driver.erl index ff2e0eb..5314cfb 100644 --- a/src/fabric_swift_driver.erl +++ b/src/fabric_swift_driver.erl @@ -13,34 +13,42 @@ -module(fabric_swift_driver). -export([put_object/5, delete_object/2, get_object/2, head_object/2]). --export([store_id/0, create_container/1, delete_container/1 ]). +-export([store_id/0, create_dataroot/1, delete_dataroot/1 ]). %% ==================================================================== %% OpenStack Swift implementation %% ==================================================================== +-define(l2b(V), list_to_binary(V)). + store_id() -> "swift". put_object(Container, ObjName, ContextType, CustomHeaders, Data) -> - couch_log:debug("Swift: PUT ~p/~s object ~n", [Container, ObjName]), + couch_log:info("Swift: PUT ~p/~s object ~n", [Container, ObjName]), case authenticate() of {ok, {Url, Storage_Token}} -> + couch_log:info("token: ~p~n",[Storage_Token]), ObjNameEncoded = couch_util:url_encode(ObjName), Headers = CustomHeaders ++ [{"X-Auth-Token", Storage_Token}, + %{"Transfer-Encoding", "chunked"}, {"Content-Type", unicode:characters_to_list(ContextType)}], Method = put, - couch_log:debug("Swift: PUT : ~p ~p ~p ~p~n", + couch_log:info("Swift: PUT : ~p ~p ~p ~p~n", [Url,ContextType, ObjNameEncoded, Container]), NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: PUT url: ~p~n",[NewUrl]), - R = ibrowse:send_req(NewUrl, Headers, Method, Data), - case R of - {ok, ReturnCode, _, _} -> - {ok, {ReturnCode, NewUrl}}; + couch_log:info("Swift: PUT url: ~p~n",[NewUrl]), + couch_log:info("~p ~n",[Data]), + case ibrowse:send_req(NewUrl, Headers, Method, Data) of + {ok, "201", _, _} -> + {ok, NewUrl}; + {ok, "202", _, _} -> + {ok, NewUrl}; + {error, {'EXIT', {normal, _}}} -> + {error, "PUT object failed"}; Error -> - couch_log:debug("PUT object failed ~p ~n", [element(2, Error)]), - {error, "PUT object failed"} + couch_log:info("PUT object failed ~p ~n", [element(2, Error)]), + {error, "PUT object failed"} end; {not_authenticated, _} -> {error, "Not authenticated"} @@ -48,7 +56,7 @@ put_object(Container, ObjName, ContextType, CustomHeaders, Data) -> delete_object(Container, ObjName) -> %TO-DO: should be called during database compaction. - couch_log:debug("Swift: Delete ~p/~p ~n", [Container, ObjName]), + couch_log:info("Swift: Delete ~p/~p ~n", [Container, ObjName]), case authenticate() of {ok, {Url, Storage_Token}} -> Header = [{"X-Auth-Token", Storage_Token}], @@ -56,16 +64,16 @@ delete_object(Container, ObjName) -> Method = delete, NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url ~p~n", [NewUrl]), + couch_log:info("Swift: url ~p~n", [NewUrl]), R = ibrowse:send_req(NewUrl, Header, Method), case R of {ok, ReturnCode, _, _} -> - couch_log:debug("Swift: Delete ~p. Return code ~p ~n", + couch_log:info("Swift: Delete ~p. Return code ~p ~n", [NewUrl, ReturnCode]), {ok, ReturnCode}; Error -> - couch_log:debug("Swift: Delete ~p failed. ~n", [NewUrl]), - couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + couch_log:info("Swift: Delete ~p failed. ~n", [NewUrl]), + couch_log:info("Swift: ~p ~n", [element(2, Error)]), {error, "Delete object failed"} end; {not_authenticated, _} -> @@ -73,7 +81,7 @@ delete_object(Container, ObjName) -> end. get_object(Container, ObjName) -> - couch_log:debug("Swift: get object ~p/~p ~n", [Container, ObjName]), + couch_log:info("Swift: get object ~p/~p ~n", [Container, ObjName]), case authenticate() of {ok, {Url, Storage_Token}} -> Header = [{"X-Auth-Token", Storage_Token}], @@ -81,15 +89,15 @@ get_object(Container, ObjName) -> Method = get, NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url ~p~n", [NewUrl]), + couch_log:info("Swift: url ~p~n", [NewUrl]), R = ibrowse:send_req(NewUrl, Header, Method), case R of {ok, ReturnCode, _, Body} -> - couch_log:debug("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]), - Body; + couch_log:info("Swift: GET ~p with return code ~p ~n", [NewUrl, ReturnCode]), + {ok, Body}; Error -> - couch_log:debug("Swift: GET ~p failed. ~n", [NewUrl]), - couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + couch_log:info("Swift: GET ~p failed. ~n", [NewUrl]), + couch_log:info("Swift: ~p ~n", [element(2, Error)]), {error, "Get object failed"} end; {not_authenticated, _} -> @@ -97,7 +105,7 @@ get_object(Container, ObjName) -> end. head_object(Container, ObjName) -> - couch_log:debug("Swift: head object ~p/~p ~n", [Container, ObjName]), + couch_log:info("Swift: head object ~p/~p ~n", [Container, ObjName]), case authenticate() of {ok, {Url, Storage_Token}} -> Header = [{"X-Auth-Token", Storage_Token}], @@ -105,53 +113,56 @@ head_object(Container, ObjName) -> ObjNameEncoded = couch_util:url_encode(ObjName), NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++ "/" ++ unicode:characters_to_list(ObjNameEncoded), - couch_log:debug("Swift: url ~p~n", [NewUrl]), + couch_log:info("Swift: url ~p~n", [NewUrl]), R = ibrowse:send_req(NewUrl, Header, Method), case R of {ok, ReturnCode, Head, _} -> - couch_log:debug("Swift: ~p~p~n", [ReturnCode, Head]), + couch_log:info("Swift: ~p~p~n", [ReturnCode, Head]), ObjectSize = lists:filter(fun ({"Content-Length", _}) -> true; (_) -> false end, Head), EtagHeader = lists:filter(fun ({"Etag", _}) -> true ; (_) -> false end, Head), Etag = element(2, lists:nth(1, EtagHeader)), {ObjectSizeNumeric, _} = string:to_integer(element(2, lists:nth(1, ObjectSize))), - couch_log:debug("Swift: Object size is: ~p and Etag: ~p~n", + couch_log:info("Swift: Object size is: ~p and Etag: ~p~n", [ObjectSizeNumeric, Etag]), EtagDecode = hex_to_bin(Etag), EtagMD5 = base64:encode(EtagDecode), - couch_log:debug("Etag in base64 ~p~n", [EtagMD5]), + couch_log:info("Etag in base64 ~p~n", [EtagMD5]), {ok, {ObjectSizeNumeric, EtagMD5}}; Error -> - couch_log:debug("Swift: Head ~p failed ~n", [NewUrl]), - couch_log:debug("Swift: ~p ~n", [element(2, Error)]), + couch_log:info("Swift: Head ~p failed ~n", [NewUrl]), + couch_log:info("Swift: ~p ~n", [element(2, Error)]), {error,"HEAD object failed"} end; {not_authenticated, _} -> {error, "Not authenticated"} end. -create_container(DbName) -> - couch_log:debug("Swift : create container ~p~n", [DbName]), +create_dataroot(DbName) -> + couch_log:info("Swift : create container ~p~n", [DbName]), case authenticate() of {ok, {Url, Storage_Token}} -> Header = [{"X-Auth-Token",Storage_Token}], Method = put, Container = DbName, NewUrl = Url ++ "/" ++ unicode:characters_to_list(Container) ++"/", - couch_log:debug("Swift: url ~p ~n",[NewUrl]), + couch_log:info("Swift: url ~p ~n",[NewUrl]), R = ibrowse:send_req(NewUrl, Header, Method), case R of - {ok, ReturnCode, _, _} -> - couch_log:debug("Swift: container ~p created with code : ~p~n",[Container, ReturnCode]), - {ok,{ReturnCode,Container}}; + {ok, "201", _, _} -> + couch_log:info("Swift: container ~p created with code : 201 ~n",[Container]), + {ok, Container}; + {ok, "202", _, _} -> + couch_log:info("Swift: container ~p created with code : 202 ~n",[Container]), + {ok, Container}; Error -> - couch_log:debug("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]), + couch_log:info("Swift: container ~p creation failed : ~p~n",[Container, element(2,Error)]), {error, "Failed to create container"} end; {not_authenticated,_} -> {error, "Failed to create container. Not authenticated"} end. -delete_container(Container) -> +delete_dataroot(Container) -> case authenticate() of {ok,{Url,Storage_Token}} -> Header = [{"X-Auth-Token",Storage_Token}], @@ -160,14 +171,14 @@ delete_container(Container) -> R = ibrowse:send_req(NewUrl, Header, Method), case R of {ok, ReturnCode, _, _} -> - {ok,{ReturnCode,Container}}; + {ok, {ReturnCode,Container}}; Error -> - {error,{element(2,Error),Container}} + {error, {element(2, Error), Container}} end; {not_authenticated, _} -> {error, "Not authenticated"}; {not_supported, Error} -> - {error,{element(2,Error),""}} + {error, {element(2, Error), ""}} end. %% ==================================================================== @@ -177,14 +188,14 @@ delete_container(Container) -> hex_to_bin(Str) -> << << (erlang:list_to_integer([H], 16)):4 >> || H <- Str >>. authenticate() -> - couch_log:debug("Going to authenticate in Swift",[]), + couch_log:info("Going to authenticate in Swift",[]), case config:get("swift", "auth_model", "tempauth") of "tempauth" -> swift_v1_auth(); "keystone" -> keystone_auth(); Error -> - couch_log:debug("Authentication method is not supported: ~p", element(2,Error)), + couch_log:info("Authentication method is not supported: ~p", element(2,Error)), {not_authenticated, ""} end. @@ -192,26 +203,40 @@ keystone_auth() -> Method = post, URL = config:get("swift", "auth_url"), Header = [{"Content-Type", "application/json"}], - Tenant = config:get("swift", "account_tenant"), - User = config:get("swift", "username"), - Psw = config:get("swift", "password"), - Body = "{\"auth\": {\"tenantName\": \"" ++ Tenant - ++ "\", \"passwordCredentials\": {\"username\": \"" - ++ User ++ "\", \"password\": \"" ++ Psw ++ "\"}}}", + Tenant = ?l2b(config:get("swift", "account_tenant")), + User = ?l2b(config:get("swift", "username")), + Psw = ?l2b(config:get("swift", "password")), + BodyValue = { + [ + {auth, {[ + {tenantName, Tenant}, + {passwordCredentials, {[ + {username, User}, + {password, Psw} + ]} + } + ]} + }]}, + Body = jiffy:encode(BodyValue, [force_utf8]), + couch_log:info("~p~n", [Body]), R = ibrowse:send_req(URL, Header, Method, Body), {ok, ReturnCode, _, Response} = R, case ReturnCode of "200" -> - couch_log:debug("keystone: authenticated: ~p~n", [ReturnCode]), + couch_log:info("keystone: authenticated: ~p~n", [ReturnCode]), Json = jiffy:decode(Response), + {Json1} = Json, + {Access} = proplists:get_value(<<"access">>, Json1), + {Token} = proplists:get_value(<<"token">>, Access), + TokenID = proplists:get_value(<<"id">>, Token), AuthInfoBinary = extractAuthInfo(Json, [{<<"access">>, ""}, {<<"serviceCatalog">>, ""}, {<<"name">>, <<"swift">>}, {<<"endpoints">>, ""}]), {ok, {binary_to_list(lists:nth(1, AuthInfoBinary)), - binary_to_list(lists:nth(2, AuthInfoBinary))}}; + binary_to_list(TokenID)}}; _ -> - couch_log:debug("keystone: not authenticated: ~p~n", [ReturnCode]), + couch_log:info("keystone: not authenticated: ~p~n", [ReturnCode]), {not_authenticated, ""} end. @@ -225,14 +250,14 @@ swift_v1_auth() -> {ok, ReturnCode, Head, _} = R, case ReturnCode of "200" -> - couch_log:debug("swift v1: authenticated: ~p~n", [ReturnCode]), + couch_log:info("swift v1: authenticated: ~p~n", [ReturnCode]), Storage_URL_F = fun ({"X-Storage-Url", _}) -> true ; (_) -> false end, Storage_URL = lists:filter(Storage_URL_F, Head), Storage_Token_F = fun ({"X-Storage-Token", _}) -> true ; (_) -> false end, Storage_Token = lists:filter(Storage_Token_F, Head), - {ok,{element(2,lists:nth(1, Storage_URL)), element(2,lists:nth(1, Storage_Token))}}; + {ok, {element(2, lists:nth(1, Storage_URL)), element(2, lists:nth(1, Storage_Token))}}; _ -> - couch_log:debug("swift v1: not authenticated: ~p~n", [ReturnCode]), + couch_log:info("swift v1: not authenticated: ~p~n", [ReturnCode]), {not_authenticated, ReturnCode} end.