Skip to content

Commit 34d8321

Browse files
kuenishikuenishi
committed
Merge pull request #759 from basho/bugfix/ku-cs-743-timeout
handle timeout error on sum_bucket/2
2 parents a06554e + 22c84d3 commit 34d8321

4 files changed

Lines changed: 146 additions & 24 deletions

File tree

‎riak_test/src/rtcs.erl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -625,3 +625,21 @@ make_authorization(Method, Resource, ContentType, Config, Date) ->
625625
StringToSign = [Method, $\n, [], $\n, ContentType, $\n, Date, $\n, Resource],
626626
Signature = base64:encode_to_string(crypto:sha_mac(Config#aws_config.secret_access_key, StringToSign)),
627627
lists:flatten(["AWS ", Config#aws_config.access_key_id, $:, Signature]).
628+
629+
datetime() ->
630+
{{YYYY,MM,DD}, {H,M,S}} = calendar:universal_time(),
631+
io_lib:format("~4..0B~2..0B~2..0BT~2..0B~2..0B~2..0BZ", [YYYY, MM, DD, H, M, S]).
632+
633+
634+
635+
json_get(Key, Json) when is_binary(Key) ->
636+
json_get([Key], Json);
637+
json_get([], Json) ->
638+
Json;
639+
json_get([Key | Keys], {struct, JsonProps}) ->
640+
case lists:keyfind(Key, 1, JsonProps) of
641+
false ->
642+
notfound;
643+
{Key, Value} ->
644+
json_get(Keys, Value)
645+
end.

‎riak_test/tests/access_stats_test.erl‎

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ confirm() ->
4141
pass.
4242

4343
generate_some_accesses(UserConfig) ->
44-
Begin = datetime(),
44+
Begin = rtcs:datetime(),
4545
lager:info("creating bucket ~p", [?BUCKET]),
4646
%% Create bucket
4747
?assertEqual(ok, erlcloud_s3:create_bucket(?BUCKET, UserConfig)),
@@ -57,7 +57,7 @@ generate_some_accesses(UserConfig) ->
5757
_ = erlcloud_s3:delete_object(?BUCKET, ?KEY, UserConfig),
5858
%% Delete bucket
5959
?assertEqual(ok, erlcloud_s3:delete_bucket(?BUCKET, UserConfig)),
60-
End = datetime(),
60+
End = rtcs:datetime(),
6161
{Begin, End}.
6262

6363
flush_access_stats() ->
@@ -88,40 +88,24 @@ assert_access_stats(UserConfig, {Begin, End}) ->
8888
?assertEqual( 1, sum_samples([<<"BucketDelete">>, <<"Count">>], Samples)),
8989
pass.
9090

91-
datetime() ->
92-
{{YYYY,MM,DD}, {H,M,S}} = calendar:universal_time(),
93-
io_lib:format("~4..0B~2..0B~2..0BT~2..0B~2..0B~2..0BZ", [YYYY, MM, DD, H, M, S]).
94-
9591
node_samples_from_usage(Node, Usage) ->
96-
ListOfNodeStats = json_get([<<"Access">>, <<"Nodes">>], Usage),
92+
ListOfNodeStats = rtcs:json_get([<<"Access">>, <<"Nodes">>], Usage),
9793
lager:debug("ListOfNodeStats: ~p", [ListOfNodeStats]),
9894
[NodeStats | _] = lists:dropwhile(fun(NodeStats) ->
99-
json_get(<<"Node">>, NodeStats) =/= Node
95+
rtcs:json_get(<<"Node">>, NodeStats) =/= Node
10096
end, ListOfNodeStats),
101-
json_get(<<"Samples">>, NodeStats).
97+
rtcs:json_get(<<"Samples">>, NodeStats).
10298

10399
%% Sum up statistics entries in possibly multiple samples
104100
sum_samples(Keys, Samples) ->
105101
sum_samples(Keys, Samples, 0).
106102
sum_samples(_Keys, [], Sum) ->
107103
Sum;
108104
sum_samples(Keys, [Sample | Samples], Sum) ->
109-
InSample = case json_get(Keys, Sample) of
105+
InSample = case rtcs:json_get(Keys, Sample) of
110106
notfound ->
111107
0;
112108
Value when is_integer(Value) ->
113109
Value
114110
end,
115111
sum_samples(Keys, Samples, Sum + InSample).
116-
117-
json_get(Key, Json) when is_binary(Key) ->
118-
json_get([Key], Json);
119-
json_get([], Json) ->
120-
Json;
121-
json_get([Key | Keys], {struct, JsonProps}) ->
122-
case lists:keyfind(Key, 1, JsonProps) of
123-
false ->
124-
notfound;
125-
{Key, Value} ->
126-
json_get(Keys, Value)
127-
end.

‎riak_test/tests/cs743_regression_test.erl‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
%% ---------------------------------------------------------------------
2+
%%
3+
%% Copyright (c) 2007-2013 Basho Technologies, Inc. All Rights Reserved.
4+
%%
5+
%% This file is provided to you under the Apache License,
6+
%% Version 2.0 (the "License"); you may not use this file
7+
%% except in compliance with the License. You may obtain
8+
%% a copy of the License at
9+
%%
10+
%% http://www.apache.org/licenses/LICENSE-2.0
11+
%%
12+
%% Unless required by applicable law or agreed to in writing,
13+
%% software distributed under the License is distributed on an
14+
%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
%% KIND, either express or implied. See the License for the
16+
%% specific language governing permissions and limitations
17+
%% under the License.
18+
%%
19+
%% ---------------------------------------------------------------------
20+
21+
-module(cs743_regression_test).
22+
23+
%% @doc Regression test for `riak_cs' <a href="https://github.com/basho/riak_cs/issues/286">
24+
%% issue 286</a>.
25+
26+
-export([confirm/0]).
27+
28+
-include_lib("erlcloud/include/erlcloud_aws.hrl").
29+
-include_lib("eunit/include/eunit.hrl").
30+
31+
-define(TEST_BUCKET, "riak-test-bucket").
32+
33+
confirm() ->
34+
Config = [{riak, rtcs:riak_config()}, {stanchion, rtcs:stanchion_config()},
35+
{cs,
36+
[{riakc, [{mapred_timeout,1}]}] %% make storage calc timeout
37+
++ rtcs:cs_config([{storage_archive_period, 1}])}],
38+
{UserConfig, {_RiakNodes, CSNodes, _Stanchion}} = rtcs:setup(4, Config),
39+
40+
Begin = rtcs:datetime(),
41+
run_storage_batch(hd(CSNodes)),
42+
lager:info("creating bucket ~p", [?TEST_BUCKET]),
43+
?assertEqual(ok, erlcloud_s3:create_bucket(?TEST_BUCKET, UserConfig)),
44+
45+
N = 1024,
46+
lager:info("creating ~p objects in ~p", [N, ?TEST_BUCKET]),
47+
ok = etoomanyobjects(N, UserConfig),
48+
timer:sleep(1000),
49+
50+
run_storage_batch(hd(CSNodes)),
51+
timer:sleep(1000),
52+
End = rtcs:datetime(),
53+
54+
assert_storage_stats(UserConfig, Begin, End),
55+
pass.
56+
57+
assert_storage_stats(UserConfig, Begin, End) ->
58+
KeyId = UserConfig#aws_config.access_key_id,
59+
StatsKey = lists:flatten(["usage/", KeyId, "/bj/", Begin, "/", End, "/"]),
60+
GetResult = erlcloud_s3:get_object("riak-cs", StatsKey, UserConfig),
61+
lager:info("Storage stats response: ~p", [GetResult]),
62+
Usage = mochijson2:decode(proplists:get_value(content, GetResult)),
63+
lager:info("Storage Usage: ~p", [Usage]),
64+
Samples = rtcs:json_get([<<"Storage">>, <<"Samples">>], Usage),
65+
66+
?assert(lists:any(
67+
fun(Sample) ->
68+
case rtcs:json_get(list_to_binary(?TEST_BUCKET), Sample) of
69+
notfound -> false;
70+
ErrorStr ->
71+
?assert(not is_integer(ErrorStr)),
72+
?assertEqual(<<"{error,{timeout,[]}}">>, ErrorStr),
73+
true
74+
end
75+
end,
76+
Samples)).
77+
%% supposed to be "{error, timeout}"
78+
79+
run_storage_batch(CSNode) ->
80+
{ok, Status0} = rpc:call(CSNode, riak_cs_storage_d, status, []),
81+
lager:info("~p", [Status0]),
82+
ok = rpc:call(CSNode, riak_cs_storage_d, start_batch, [[{recalc,true}]]),
83+
{ok, Status1} = rpc:call(CSNode, riak_cs_storage_d, status, []),
84+
lager:info("~p", [Status1]),
85+
%%{ok,
86+
%% {calculating,[{schedule,[]},{last,undefined},{current,{{2013,12,26},{3,55,29}}},
87+
%% {next,undefined},{elapsed,0},{users_done,1},{users_skipped,0},{users_left,0}]}}
88+
89+
{_Status, Result} = Status1,
90+
1 = proplists:get_value(users_done,Result),
91+
0 = proplists:get_value(users_skipped,Result),
92+
0 = proplists:get_value(users_left,Result).
93+
94+
etoomanyobjects(N, UserConfig) ->
95+
SingleBlock = crypto:rand_bytes(400),
96+
lists:map(fun(I) ->
97+
R = erlcloud_s3:put_object(?TEST_BUCKET, integer_to_list(I),
98+
SingleBlock, UserConfig),
99+
[{version_id,"null"}] = R
100+
end,
101+
lists:seq(1,N)),
102+
ok.

‎src/riak_cs_storage.erl‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,30 @@ sum_user(Riak, User) when is_binary(User) ->
4646
sum_user(Riak, User) when is_list(User) ->
4747
case riak_cs_utils:get_user(User, Riak) of
4848
{ok, {?RCS_USER{buckets=Buckets}, _UserObj}} ->
49-
{ok, [ {B, sum_bucket(Riak, B)}
50-
|| ?RCS_BUCKET{name=B} <- Buckets ]};
49+
BucketUsages = [maybe_sum_bucket(Riak, User, B)
50+
|| ?RCS_BUCKET{name=B} <- Buckets],
51+
{ok, BucketUsages};
5152
{error, Error} ->
5253
{error, Error}
5354
end.
5455

56+
%% @doc Output a log when calculating total usage of a bucket.
57+
%% This log is *very* important because unless this log
58+
%% there are no other way for operator to know a calculation
59+
%% which riak_cs_storage_d failed.
60+
-spec maybe_sum_bucket(pid(), string(), string()) ->
61+
{string(), term()|string()}.
62+
maybe_sum_bucket(Riak, User, Bucket) ->
63+
case sum_bucket(Riak, Bucket) of
64+
{struct, _} = BucketUsage -> BucketUsage;
65+
66+
{error, _} = E ->
67+
_ = lager:error("failed to calculate usage of "
68+
"bucket '~s' of user '~s'. Reason: ~p",
69+
[Bucket, User, E]),
70+
{Bucket, iolist_to_binary(io_lib:format("~p", [E]))}
71+
end.
72+
5573
%% @doc Sum the number of bytes stored in active files in the named
5674
%% bucket. This assumes that the bucket exists; there will be no
5775
%% difference in output between a non-existent bucket and an empty

0 commit comments

Comments
 (0)