diff options
| -rw-r--r-- | src/fsdb.erl | 157 | ||||
| -rw-r--r-- | src/perm.erl | 155 | ||||
| -rw-r--r-- | src/permdb.erl | 5 | ||||
| -rw-r--r-- | src/plop_app.erl | 2 |
4 files changed, 179 insertions, 140 deletions
diff --git a/src/fsdb.erl b/src/fsdb.erl new file mode 100644 index 0000000..3e1b9b9 --- /dev/null +++ b/src/fsdb.erl @@ -0,0 +1,157 @@ +%%% Copyright (c) 2014-2015, NORDUnet A/S. +%%% See LICENSE for licensing information. + +-module(fsdb). +-behaviour(gen_server). + +-export([start_link/2, stop/1, init_module/0]). +-export([getvalue/2, addvalue/3, commit/1, commit/2]). + +%% gen_server callbacks. +-export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, + code_change/3]). + +-record(state, {name, dirtylistname}). + +-define(DIRECTORY_TABLE, perm_directory). + +init_module() -> + case ets:info(?DIRECTORY_TABLE) of + undefined -> ok; + _ -> ets:delete(?DIRECTORY_TABLE) + end, + ets:new(?DIRECTORY_TABLE, [set, public, named_table]). + +start_link(Name, Filename) -> + gen_server:start_link({local, Name}, ?MODULE, + [Name, Filename], []). + +stop(Name) -> + gen_server:call(Name, stop). + +addfsyncfiles(Name, Files) -> + gen_server:call(Name, {addfsyncfiles, Files}). + +init([Name, Filename]) -> + Dirtylistname = list_to_atom(atom_to_list(Name) ++ "_dirty"), + true = ets:insert(?DIRECTORY_TABLE, {Name, Filename ++ "/"}), + ets:new(Dirtylistname, [set, public, named_table]), + {ok, #state{name = Name, dirtylistname = Dirtylistname}}. + +handle_cast(_Request, State) -> + {noreply, State}. + +handle_info(_Info, State) -> + {noreply, State}. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +terminate(_Reason, _State) -> + io:format("~p terminating~n", [?MODULE]), + ok. + +handle_call({addfsyncfiles, Files}, _From, State) -> + Insert = lists:map(fun (File) -> {File} end, Files), + ets:insert(State#state.dirtylistname, Insert), + {reply, ok, State}; +handle_call({commit, Timeout}, _From, State) -> + lager:debug("doing commit for ~p", [State#state.name]), + Files = lists:map(fun ([File]) -> File end, ets:match(State#state.dirtylistname, {'$1'})), + util:fsync(Files, Timeout), + ets:delete_all_objects(State#state.dirtylistname), + {reply, ok, State}; +handle_call(stop, _From, State) -> + {stop, normal, stopped, State}. + +getvalue(Name, Key) -> + [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name), + readfile(Filename, Key). + +addvalue(Name, Key, Value) -> + [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name), + {Result, FsyncFiles} = ensurefile(Filename, Key, Value), + addfsyncfiles(Name, FsyncFiles), + Result. + +commit(Name) -> + commit(Name, 5000). + +commit(Name, Timeout) -> + gen_server:call(Name, {commit, Timeout}, Timeout). + +-spec readfile_and_verify(string(), binary()) -> ok | differ | {error, atom()}. +readfile_and_verify(Name, Content) -> + case file:read_file(Name) of + {ok, ContentsRead} when Content == ContentsRead -> + ok; + {ok, _ContentsRead} -> + differ; + {error, Error} -> + {error, Error} + end. + +-spec make_dir(string()) -> ok | {error, atom()}. +make_dir(Name) -> + case file:make_dir(Name) of + ok -> + ok; + {error, eexist} -> + ok; + {error, Error} -> + {error, Error} + end. + +-spec make_dirs([string()]) -> ok | {error, atom()}. +make_dirs([]) -> + ok; +make_dirs([Name | Rest]) -> + case make_dir(Name) of + ok -> + make_dirs(Rest); + {error, Error} -> + {error, Error} + end. + +-spec path_for_key(string(), binary()) -> {[string()], string()}. +path_for_key(Rootdir, Key) -> + Name = hex:bin_to_hexstr(Key), + [C1, C2, C3, C4, C5, C6 | _] = Name, + Firstlevel = Rootdir ++ [C1, C2], + Secondlevel = Firstlevel ++ "/" ++ [C3, C4], + Thirdlevel = Secondlevel ++ "/" ++ [C5, C6], + Fullpath = Thirdlevel ++ "/" ++ Name, + {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. + +ensurefile(Rootdir, Key, Content) -> + lager:debug("dir ~p key ~s", [Rootdir, mochihex:to_hex(Key)]), + {Dirs, Path} = path_for_key(Rootdir, Key), + Result = + case readfile_and_verify(Path, Content) of + ok -> + lager:debug("key ~s existed, fsync", [mochihex:to_hex(Key)]), + ok = util:fsync([Path, Rootdir | Dirs]), + lager:debug("key ~s fsynced", [mochihex:to_hex(Key)]), + ok; + differ -> + lager:debug("key ~s existed, was different", + [mochihex:to_hex(Key)]), + differ; + {error, enoent} -> + lager:debug("key ~s didn't exist, add", [mochihex:to_hex(Key)]), + util:check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] + ++ Dirs), + makedir, "Error creating directory"), + NurseryName = Rootdir ++ "nursery/" ++ + util:tempfilename(hex:bin_to_hexstr(Key)), + util:write_tempfile_and_rename(Path, NurseryName, Content), + ok; + {error, Error} -> + util:exit_with_error(Error, readfile, "Error reading file") + end, + {Result, [Path, Rootdir | Dirs]}. + +-spec readfile(string(), binary()) -> binary() | noentry. +readfile(Rootdir, Key) -> + {_Dirs, Path} = path_for_key(Rootdir, Key), + atomic:readfile(Path). diff --git a/src/perm.erl b/src/perm.erl index 6c62b59..2e12fdf 100644 --- a/src/perm.erl +++ b/src/perm.erl @@ -1,157 +1,36 @@ -%%% Copyright (c) 2014-2015, NORDUnet A/S. +%%% Copyright (c) 2015, NORDUnet A/S. %%% See LICENSE for licensing information. -module(perm). --behaviour(gen_server). --export([start_link/2, stop/1, init_directory_table/0]). +-export([start_link/2, stop/1, init_module/0]). -export([getvalue/2, addvalue/3, commit/1, commit/2]). -%% gen_server callbacks. --export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2, - code_change/3]). - --record(state, {name, dirtylistname}). - --define(DIRECTORY_TABLE, perm_directory). - -init_directory_table() -> - case ets:info(?DIRECTORY_TABLE) of - undefined -> ok; - _ -> ets:delete(?DIRECTORY_TABLE) - end, - ets:new(?DIRECTORY_TABLE, [set, public, named_table]). - start_link(Name, Filename) -> - gen_server:start_link({local, Name}, ?MODULE, - [Name, Filename], []). + Module = application:get_env(plop, db_backend, fsdb), + Module:start_link(Name, Filename). stop(Name) -> - gen_server:call(Name, stop). - -addfsyncfiles(Name, Files) -> - gen_server:call(Name, {addfsyncfiles, Files}). - -init([Name, Filename]) -> - Dirtylistname = list_to_atom(atom_to_list(Name) ++ "_dirty"), - true = ets:insert(?DIRECTORY_TABLE, {Name, Filename}), - ets:new(Dirtylistname, [set, public, named_table]), - {ok, #state{name = Name, dirtylistname = Dirtylistname}}. - -handle_cast(_Request, State) -> - {noreply, State}. - -handle_info(_Info, State) -> - {noreply, State}. + Module = application:get_env(plop, db_backend, fsdb), + Module:stop(Name). -code_change(_OldVsn, State, _Extra) -> - {ok, State}. - -terminate(_Reason, _State) -> - io:format("~p terminating~n", [?MODULE]), - ok. - -handle_call({addfsyncfiles, Files}, _From, State) -> - Insert = lists:map(fun (File) -> {File} end, Files), - ets:insert(State#state.dirtylistname, Insert), - {reply, ok, State}; -handle_call({commit, Timeout}, _From, State) -> - lager:debug("doing commit for ~p", [State#state.name]), - Files = lists:map(fun ([File]) -> File end, ets:match(State#state.dirtylistname, {'$1'})), - util:fsync(Files, Timeout), - ets:delete_all_objects(State#state.dirtylistname), - {reply, ok, State}; -handle_call(stop, _From, State) -> - {stop, normal, stopped, State}. +init_module() -> + Module = application:get_env(plop, db_backend, fsdb), + Module:init_module(). getvalue(Name, Key) -> - [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name), - readfile(Filename, Key). + Module = application:get_env(plop, db_backend, fsdb), + Module:getvalue(Name, Key). addvalue(Name, Key, Value) -> - [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name), - {Result, FsyncFiles} = ensurefile(Filename, Key, Value), - addfsyncfiles(Name, FsyncFiles), - Result. + Module = application:get_env(plop, db_backend, fsdb), + Module:addvalue(Name, Key, Value). commit(Name) -> - commit(Name, 5000). + Module = application:get_env(plop, db_backend, fsdb), + Module:commit(Name). commit(Name, Timeout) -> - gen_server:call(Name, {commit, Timeout}, Timeout). - --spec readfile_and_verify(string(), binary()) -> ok | differ | {error, atom()}. -readfile_and_verify(Name, Content) -> - case file:read_file(Name) of - {ok, ContentsRead} when Content == ContentsRead -> - ok; - {ok, _ContentsRead} -> - differ; - {error, Error} -> - {error, Error} - end. - --spec make_dir(string()) -> ok | {error, atom()}. -make_dir(Name) -> - case file:make_dir(Name) of - ok -> - ok; - {error, eexist} -> - ok; - {error, Error} -> - {error, Error} - end. - --spec make_dirs([string()]) -> ok | {error, atom()}. -make_dirs([]) -> - ok; -make_dirs([Name | Rest]) -> - case make_dir(Name) of - ok -> - make_dirs(Rest); - {error, Error} -> - {error, Error} - end. - --spec path_for_key(string(), binary()) -> {[string()], string()}. -path_for_key(Rootdir, Key) -> - Name = hex:bin_to_hexstr(Key), - [C1, C2, C3, C4, C5, C6 | _] = Name, - Firstlevel = Rootdir ++ [C1, C2], - Secondlevel = Firstlevel ++ "/" ++ [C3, C4], - Thirdlevel = Secondlevel ++ "/" ++ [C5, C6], - Fullpath = Thirdlevel ++ "/" ++ Name, - {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}. - -ensurefile(Rootdir, Key, Content) -> - lager:debug("dir ~p key ~s", [Rootdir, mochihex:to_hex(Key)]), - {Dirs, Path} = path_for_key(Rootdir, Key), - Result = - case readfile_and_verify(Path, Content) of - ok -> - lager:debug("key ~s existed, fsync", [mochihex:to_hex(Key)]), - ok = util:fsync([Path, Rootdir | Dirs]), - lager:debug("key ~s fsynced", [mochihex:to_hex(Key)]), - ok; - differ -> - lager:debug("key ~s existed, was different", - [mochihex:to_hex(Key)]), - differ; - {error, enoent} -> - lager:debug("key ~s didn't exist, add", [mochihex:to_hex(Key)]), - util:check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"] - ++ Dirs), - makedir, "Error creating directory"), - NurseryName = Rootdir ++ "nursery/" ++ - util:tempfilename(hex:bin_to_hexstr(Key)), - util:write_tempfile_and_rename(Path, NurseryName, Content), - ok; - {error, Error} -> - util:exit_with_error(Error, readfile, "Error reading file") - end, - {Result, [Path, Rootdir | Dirs]}. + Module = application:get_env(plop, db_backend, fsdb), + Module:commit(Name, Timeout). --spec readfile(string(), binary()) -> binary() | noentry. -readfile(Rootdir, Key) -> - {_Dirs, Path} = path_for_key(Rootdir, Key), - atomic:readfile(Path). diff --git a/src/permdb.erl b/src/permdb.erl index 0cb3c2f..cc878f4 100644 --- a/src/permdb.erl +++ b/src/permdb.erl @@ -5,7 +5,7 @@ -behaviour(gen_server). --export([start_link/2, stop/1]). +-export([start_link/2, stop/1, init_module/0]). -export([getvalue/2, addvalue/3, commit/1, commit/2]). %% gen_server callbacks. @@ -127,6 +127,9 @@ init([Name, Filename]) -> indexfile = IndexFile, requests = queue:new()}}. +init_module() -> + ok. + start_link(Name, Filename) -> gen_server:start_link({local, Name}, ?MODULE, [Name, Filename], []). diff --git a/src/plop_app.erl b/src/plop_app.erl index e154555..dc896e2 100644 --- a/src/plop_app.erl +++ b/src/plop_app.erl @@ -9,7 +9,7 @@ start(normal, Args) -> hackney:start(), http_auth:init_key_table(), plop:initsize(), - perm:init_directory_table(), + perm:init_module(), plop_sup:start_link(Args). stop(_State) -> |