Make it possible to select backend perm storage

1 files changed, 157 insertions, 0 deletions

diff --git a/src/fsdb.erl b/src/fsdb.erl
new file mode 100644
index 0000000..3e1b9b9
--- /dev/null
+++ b/src/fsdb.erl
@@ -0,0 +1,157 @@
+%%% Copyright (c) 2014-2015, NORDUnet A/S.
+%%% See LICENSE for licensing information.
+
+-module(fsdb).
+-behaviour(gen_server).
+
+-export([start_link/2, stop/1, init_module/0]).
+-export([getvalue/2, addvalue/3, commit/1, commit/2]).
+
+%% gen_server callbacks.
+-export([init/1, handle_call/3, terminate/2, handle_cast/2, handle_info/2,
+         code_change/3]).
+
+-record(state, {name, dirtylistname}).
+
+-define(DIRECTORY_TABLE, perm_directory).
+
+init_module() ->
+    case ets:info(?DIRECTORY_TABLE) of
+	undefined -> ok;
+	_ -> ets:delete(?DIRECTORY_TABLE)
+    end,
+    ets:new(?DIRECTORY_TABLE, [set, public, named_table]).
+
+start_link(Name, Filename) ->
+    gen_server:start_link({local, Name}, ?MODULE,
+                          [Name, Filename], []).
+
+stop(Name) ->
+    gen_server:call(Name, stop).
+
+addfsyncfiles(Name, Files) ->
+    gen_server:call(Name, {addfsyncfiles, Files}).
+
+init([Name, Filename]) ->
+    Dirtylistname = list_to_atom(atom_to_list(Name) ++ "_dirty"),
+    true = ets:insert(?DIRECTORY_TABLE, {Name, Filename ++ "/"}),
+    ets:new(Dirtylistname, [set, public, named_table]),
+    {ok, #state{name = Name, dirtylistname = Dirtylistname}}.
+
+handle_cast(_Request, State) ->
+    {noreply, State}.
+
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+terminate(_Reason, _State) ->
+    io:format("~p terminating~n", [?MODULE]),
+    ok.
+
+handle_call({addfsyncfiles, Files}, _From, State) ->
+    Insert = lists:map(fun (File) -> {File} end, Files),
+    ets:insert(State#state.dirtylistname, Insert),
+    {reply, ok, State};
+handle_call({commit, Timeout}, _From, State) ->
+    lager:debug("doing commit for ~p", [State#state.name]),
+    Files = lists:map(fun ([File]) -> File end, ets:match(State#state.dirtylistname, {'$1'})),
+    util:fsync(Files, Timeout),
+    ets:delete_all_objects(State#state.dirtylistname),
+    {reply, ok, State};
+handle_call(stop, _From, State) ->
+    {stop, normal, stopped, State}.
+
+getvalue(Name, Key) ->
+    [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name),
+    readfile(Filename, Key).
+
+addvalue(Name, Key, Value) ->
+    [{_, Filename}] = ets:lookup(?DIRECTORY_TABLE, Name),
+    {Result, FsyncFiles} = ensurefile(Filename, Key, Value),
+    addfsyncfiles(Name, FsyncFiles),
+    Result.
+
+commit(Name) ->
+    commit(Name, 5000).
+
+commit(Name, Timeout) ->
+    gen_server:call(Name, {commit, Timeout}, Timeout).
+
+-spec readfile_and_verify(string(), binary()) -> ok | differ | {error, atom()}.
+readfile_and_verify(Name, Content) ->
+    case file:read_file(Name) of
+        {ok, ContentsRead} when Content == ContentsRead ->
+            ok;
+        {ok, _ContentsRead} ->
+            differ;
+        {error, Error} ->
+            {error, Error}
+    end.
+
+-spec make_dir(string()) -> ok | {error, atom()}.
+make_dir(Name) ->
+    case file:make_dir(Name) of
+	ok ->
+	    ok;
+	{error, eexist} ->
+	    ok;
+	{error, Error} ->
+	    {error, Error}
+    end.
+
+-spec make_dirs([string()]) -> ok | {error, atom()}.
+make_dirs([]) ->
+    ok;
+make_dirs([Name | Rest]) ->
+    case make_dir(Name) of
+	ok ->
+	    make_dirs(Rest);
+	{error, Error} ->
+	    {error, Error}
+    end.
+
+-spec path_for_key(string(), binary()) -> {[string()], string()}.
+path_for_key(Rootdir, Key) ->
+    Name = hex:bin_to_hexstr(Key),
+    [C1, C2, C3, C4, C5, C6 | _] = Name,
+    Firstlevel = Rootdir ++ [C1, C2],
+    Secondlevel = Firstlevel ++ "/" ++ [C3, C4],
+    Thirdlevel = Secondlevel ++ "/" ++ [C5, C6],
+    Fullpath = Thirdlevel ++ "/" ++ Name,
+    {[Firstlevel, Secondlevel, Thirdlevel], Fullpath}.
+
+ensurefile(Rootdir, Key, Content) ->
+    lager:debug("dir ~p key ~s", [Rootdir, mochihex:to_hex(Key)]),
+    {Dirs, Path} = path_for_key(Rootdir, Key),
+    Result =
+        case readfile_and_verify(Path, Content) of
+            ok ->
+                lager:debug("key ~s existed, fsync", [mochihex:to_hex(Key)]),
+                ok = util:fsync([Path, Rootdir | Dirs]),
+                lager:debug("key ~s fsynced", [mochihex:to_hex(Key)]),
+                ok;
+            differ ->
+                lager:debug("key ~s existed, was different",
+                            [mochihex:to_hex(Key)]),
+                differ;
+            {error, enoent} ->
+                lager:debug("key ~s didn't exist, add", [mochihex:to_hex(Key)]),
+                util:check_error(make_dirs([Rootdir, Rootdir ++ "nursery/"]
+                                           ++ Dirs),
+                                 makedir, "Error creating directory"),
+                NurseryName = Rootdir ++ "nursery/" ++
+                    util:tempfilename(hex:bin_to_hexstr(Key)),
+                util:write_tempfile_and_rename(Path, NurseryName, Content),
+                ok;
+            {error, Error} ->
+                util:exit_with_error(Error, readfile, "Error reading file")
+        end,
+    {Result, [Path, Rootdir | Dirs]}.
+
+-spec readfile(string(), binary()) -> binary() | noentry.
+readfile(Rootdir, Key) ->
+    {_Dirs, Path} = path_for_key(Rootdir, Key),
+    atomic:readfile(Path).