From 175a76dec5bdb0a55c1fe845ad13d433b74d7d76 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 7 Jun 2024 15:46:17 -0700 Subject: [PATCH 01/54] Add Couch Stats Resource Tracker (CSRT) --- rel/overlay/etc/default.ini | 30 + src/chttpd/src/chttpd.erl | 10 + src/chttpd/src/chttpd_db.erl | 2 + src/chttpd/src/chttpd_httpd_handlers.erl | 1 + src/chttpd/src/chttpd_misc.erl | 105 ++++ .../test/eunit/chttpd_db_doc_size_tests.erl | 7 +- src/config/src/config_listener_mon.erl | 2 + src/couch/include/couch_db.hrl | 2 + src/couch/priv/stats_descriptions.cfg | 33 + src/couch/src/couch_btree.erl | 3 + src/couch/src/couch_db.erl | 2 + src/couch/src/couch_os_process.erl | 11 +- src/couch/src/couch_query_servers.erl | 8 + src/couch/src/couch_server.erl | 2 + src/couch_stats/CSRT.md | 1 + src/couch_stats/src/couch_stats.app.src | 9 +- src/couch_stats/src/couch_stats.erl | 26 + .../src/couch_stats_resource_tracker.hrl | 171 ++++++ src/couch_stats/src/couch_stats_sup.erl | 2 + src/couch_stats/src/csrt.erl | 500 +++++++++++++++ src/couch_stats/src/csrt_logger.erl | 401 ++++++++++++ src/couch_stats/src/csrt_query.erl | 174 ++++++ src/couch_stats/src/csrt_server.erl | 198 ++++++ src/couch_stats/src/csrt_util.erl | 470 ++++++++++++++ .../test/eunit/csrt_logger_tests.erl | 345 +++++++++++ .../test/eunit/csrt_server_tests.erl | 575 ++++++++++++++++++ src/fabric/priv/stats_descriptions.cfg | 50 ++ src/fabric/src/fabric_rpc.erl | 16 + src/fabric/src/fabric_util.erl | 40 +- .../test/eunit/fabric_rpc_purge_tests.erl | 2 + src/fabric/test/eunit/fabric_rpc_tests.erl | 11 +- src/ioq/src/ioq.erl | 1 + src/mango/src/mango_cursor_view.erl | 2 + src/mango/src/mango_selector.erl | 1 + src/mem3/src/mem3_rpc.erl | 34 +- src/rexi/include/rexi.hrl | 1 + src/rexi/src/rexi.erl | 10 +- src/rexi/src/rexi_monitor.erl | 1 + src/rexi/src/rexi_server.erl | 19 +- src/rexi/src/rexi_utils.erl | 12 +- src/rexi/test/rexi_tests.erl | 15 +- 41 files changed, 3261 insertions(+), 44 deletions(-) create mode 100644 src/couch_stats/CSRT.md create mode 100644 src/couch_stats/src/couch_stats_resource_tracker.hrl create mode 100644 src/couch_stats/src/csrt.erl create mode 100644 src/couch_stats/src/csrt_logger.erl create mode 100644 src/couch_stats/src/csrt_query.erl create mode 100644 src/couch_stats/src/csrt_server.erl create mode 100644 src/couch_stats/src/csrt_util.erl create mode 100644 src/couch_stats/test/eunit/csrt_logger_tests.erl create mode 100644 src/couch_stats/test/eunit/csrt_server_tests.erl diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 1a0f318bf32..a8663e3580e 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1119,3 +1119,33 @@ url = {{nouveau_url}} ;mem3_shards = true ;nouveau_index_manager = true ;dreyfus_index_manager = true + +; Couch Stats Resource Tracker (CSRT) +[csrt] +enabled = true + +; CSRT Rexi Server Init P tracking +; Enable these to enable additional metrics for RPC worker spawn rates +; Mod and Function are separated by double underscores +[csrt.init_p] +enabled = false +fabric_rpc__all_docs = true +fabric_rpc__changes = true +fabric_rpc__map_view = true +fabric_rpc__reduce_view = true +fabric_rpc__get_all_security = true +fabric_rpc__open_doc = true +fabric_rpc__update_docs = true +fabric_rpc__open_shard = true + +;; CSRT dbname matchers +;; Given a dbname and a positive integer, this will enable an IO matcher +;; against the provided db for any requests that induce IO in quantities +;; greater than the provided threshold on any one of: ioq_calls, rows_read +;; docs_read, get_kp_node, get_kv_node, or changes_processed. +;; +[csrt_logger.dbnames_io] +;; foo = 100 +;; _dbs = 123 +;; _users = 234 +;; foo/bar = 200 diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl index 57a3aeaeaa6..e9631ce48f3 100644 --- a/src/chttpd/src/chttpd.erl +++ b/src/chttpd/src/chttpd.erl @@ -339,6 +339,10 @@ handle_request_int(MochiReq) -> % Save client socket so that it can be monitored for disconnects chttpd_util:mochiweb_client_req_set(MochiReq), + %% This is probably better in before_request, but having Path is nice + csrt:create_coordinator_context(HttpReq0, Path), + csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), + {HttpReq2, Response} = case before_request(HttpReq0) of {ok, HttpReq1} -> @@ -369,6 +373,7 @@ handle_request_int(MochiReq) -> before_request(HttpReq) -> try + csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), chttpd_stats:init(), chttpd_plugin:before_request(HttpReq) catch @@ -388,6 +393,8 @@ after_request(HttpReq, HttpResp0) -> HttpResp2 = update_stats(HttpReq, HttpResp1), chttpd_stats:report(HttpReq, HttpResp2), maybe_log(HttpReq, HttpResp2), + %% NOTE: do not set_context_handler_fun to preserve the Handler + csrt:destroy_context(), HttpResp2. process_request(#httpd{mochi_req = MochiReq} = HttpReq) -> @@ -400,6 +407,7 @@ process_request(#httpd{mochi_req = MochiReq} = HttpReq) -> RawUri = MochiReq:get(raw_path), try + csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), couch_httpd:validate_host(HttpReq), check_request_uri_length(RawUri), check_url_encoding(RawUri), @@ -425,10 +433,12 @@ handle_req_after_auth(HandlerKey, HttpReq) -> HandlerKey, fun chttpd_db:handle_request/1 ), + csrt:set_context_handler_fun(HandlerFun), AuthorizedReq = chttpd_auth:authorize( possibly_hack(HttpReq), fun chttpd_auth_request:authorize_request/1 ), + csrt:set_context_username(AuthorizedReq), {AuthorizedReq, HandlerFun(AuthorizedReq)} catch ErrorType:Error:Stack -> diff --git a/src/chttpd/src/chttpd_db.erl b/src/chttpd/src/chttpd_db.erl index b4c141f8c45..2419e77d840 100644 --- a/src/chttpd/src/chttpd_db.erl +++ b/src/chttpd/src/chttpd_db.erl @@ -83,6 +83,7 @@ % Database request handlers handle_request(#httpd{path_parts = [DbName | RestParts], method = Method} = Req) -> + csrt:set_context_dbname(DbName), case {Method, RestParts} of {'PUT', []} -> create_db_req(Req, DbName); @@ -103,6 +104,7 @@ handle_request(#httpd{path_parts = [DbName | RestParts], method = Method} = Req) do_db_req(Req, fun db_req/2); {_, [SecondPart | _]} -> Handler = chttpd_handlers:db_handler(SecondPart, fun db_req/2), + csrt:set_context_handler_fun(Handler), do_db_req(Req, Handler) end. diff --git a/src/chttpd/src/chttpd_httpd_handlers.erl b/src/chttpd/src/chttpd_httpd_handlers.erl index 932b52e5f6e..e1b26022204 100644 --- a/src/chttpd/src/chttpd_httpd_handlers.erl +++ b/src/chttpd/src/chttpd_httpd_handlers.erl @@ -20,6 +20,7 @@ url_handler(<<"_utils">>) -> fun chttpd_misc:handle_utils_dir_req/1; url_handler(<<"_all_dbs">>) -> fun chttpd_misc:handle_all_dbs_req/1; url_handler(<<"_dbs_info">>) -> fun chttpd_misc:handle_dbs_info_req/1; url_handler(<<"_active_tasks">>) -> fun chttpd_misc:handle_task_status_req/1; +url_handler(<<"_active_resources">>) -> fun chttpd_misc:handle_resource_status_req/1; url_handler(<<"_scheduler">>) -> fun couch_replicator_httpd:handle_scheduler_req/1; url_handler(<<"_node">>) -> fun chttpd_node:handle_node_req/1; url_handler(<<"_reload_query_servers">>) -> fun chttpd_misc:handle_reload_query_servers_req/1; diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index 888111a64c2..0baf0b972e3 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -20,6 +20,7 @@ handle_replicate_req/1, handle_reload_query_servers_req/1, handle_task_status_req/1, + handle_resource_status_req/1, handle_up_req/1, handle_utils_dir_req/1, handle_utils_dir_req/2, @@ -219,6 +220,110 @@ handle_task_status_req(#httpd{method = 'GET'} = Req) -> handle_task_status_req(Req) -> send_method_not_allowed(Req, "GET,HEAD"). +handle_resource_status_req(#httpd{method = 'POST'} = Req) -> + ok = chttpd:verify_is_server_admin(Req), + chttpd:validate_ctype(Req, "application/json"), + {Props} = chttpd:json_body_obj(Req), + Action = proplists:get_value(<<"action">>, Props), + Key = proplists:get_value(<<"key">>, Props), + Val = proplists:get_value(<<"val">>, Props), + + CountBy = fun csrt:count_by/1, + GroupBy = fun csrt:group_by/2, + SortedBy1 = fun csrt:sorted_by/1, + SortedBy2 = fun csrt:sorted_by/2, + ConvertEle = fun erlang:binary_to_existing_atom/1, + ConvertList = fun(L) -> [ConvertEle(E) || E <- L] end, + ToJson = fun csrt_util:to_json/1, + JsonKeys = fun(PL) -> [[ToJson(K), V] || {K, V} <- PL] end, + + Fun = case {Action, Key, Val} of + {<<"count_by">>, Keys, undefined} when is_list(Keys) -> + Keys1 = [ConvertEle(K) || K <- Keys], + fun() -> CountBy(Keys1) end; + {<<"count_by">>, Key, undefined} -> + Key1 = ConvertEle(Key), + fun() -> CountBy(Key1) end; + {<<"group_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> + Keys1 = ConvertList(Keys), + Vals1 = ConvertList(Vals), + fun() -> GroupBy(Keys1, Vals1) end; + {<<"group_by">>, Key, Vals} when is_list(Vals) -> + Key1 = ConvertEle(Key), + Vals1 = ConvertList(Vals), + fun() -> GroupBy(Key1, Vals1) end; + {<<"group_by">>, Keys, Val} when is_list(Keys) -> + Keys1 = ConvertList(Keys), + Val1 = ConvertEle(Val), + fun() -> GroupBy(Keys1, Val1) end; + {<<"group_by">>, Key, Val} -> + Key1 = ConvertEle(Key), + Val1 = ConvertList(Val), + fun() -> GroupBy(Key1, Val1) end; + + {<<"sorted_by">>, Key, undefined} -> + Key1 = ConvertEle(Key), + fun() -> JsonKeys(SortedBy1(Key1)) end; + {<<"sorted_by">>, Keys, undefined} when is_list(Keys) -> + Keys1 = [ConvertEle(K) || K <- Keys], + fun() -> JsonKeys(SortedBy1(Keys1)) end; + {<<"sorted_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> + Keys1 = ConvertList(Keys), + Vals1 = ConvertList(Vals), + fun() -> JsonKeys(SortedBy2(Keys1, Vals1)) end; + {<<"sorted_by">>, Key, Vals} when is_list(Vals) -> + Key1 = ConvertEle(Key), + Vals1 = ConvertList(Vals), + fun() -> JsonKeys(SortedBy2(Key1, Vals1)) end; + {<<"sorted_by">>, Keys, Val} when is_list(Keys) -> + Keys1 = ConvertList(Keys), + Val1 = ConvertEle(Val), + fun() -> JsonKeys(SortedBy2(Keys1, Val1)) end; + {<<"sorted_by">>, Key, Val} -> + Key1 = ConvertEle(Key), + Val1 = ConvertList(Val), + fun() -> JsonKeys(SortedBy2(Key1, Val1)) end; + _ -> + throw({badrequest, invalid_resource_request}) + end, + + Fun1 = fun() -> + case Fun() of + Map when is_map(Map) -> + {maps:fold( + fun + (_K,0,A) -> A; %% TODO: Skip 0 value entries? + (K,V,A) -> [{ToJson(K), V} | A] + end, + [], Map)}; + List when is_list(List) -> + List + end + end, + + {Resp, _Bad} = rpc:multicall(erlang, apply, [ + fun() -> + {node(), Fun1()} + end, + [] + ]), + %%io:format("{CSRT}***** GOT RESP: ~p~n", [Resp]), + send_json(Req, {Resp}); +handle_resource_status_req(#httpd{method = 'GET'} = Req) -> + ok = chttpd:verify_is_server_admin(Req), + {Resp, Bad} = rpc:multicall(erlang, apply, [ + fun() -> + {node(), csrt:active()} + end, + [] + ]), + %% TODO: incorporate Bad responses + send_json(Req, {Resp}); +handle_resource_status_req(Req) -> + ok = chttpd:verify_is_server_admin(Req), + send_method_not_allowed(Req, "GET,HEAD,POST"). + + handle_replicate_req(#httpd{method = 'POST', user_ctx = Ctx, req_body = PostBody} = Req) -> chttpd:validate_ctype(Req, "application/json"), %% see HACK in chttpd.erl about replication diff --git a/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl b/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl index 01ef16f23e8..da60e85e604 100644 --- a/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl +++ b/src/chttpd/test/eunit/chttpd_db_doc_size_tests.erl @@ -24,8 +24,9 @@ setup() -> Hashed = couch_passwords:hash_admin_password(?PASS), - ok = config:set("admins", ?USER, ?b2l(Hashed), _Persist = false), - ok = config:set("couchdb", "max_document_size", "50"), + ok = config:set("admins", ?USER, ?b2l(Hashed), false), + ok = config:set("couchdb", "max_document_size", "50", false), + TmpDb = ?tempdb(), Addr = config:get("chttpd", "bind_address", "127.0.0.1"), Port = mochiweb_socket_server:get(chttpd, port), @@ -35,7 +36,7 @@ setup() -> teardown(Url) -> delete_db(Url), - ok = config:delete("admins", ?USER, _Persist = false), + ok = config:delete("admins", ?USER, false), ok = config:delete("couchdb", "max_document_size"). create_db(Url) -> diff --git a/src/config/src/config_listener_mon.erl b/src/config/src/config_listener_mon.erl index b74a6130622..898ecd183de 100644 --- a/src/config/src/config_listener_mon.erl +++ b/src/config/src/config_listener_mon.erl @@ -13,6 +13,8 @@ -module(config_listener_mon). -behaviour(gen_server). +-dialyzer({nowarn_function, init/1}). + -export([ subscribe/2, start_link/2 diff --git a/src/couch/include/couch_db.hrl b/src/couch/include/couch_db.hrl index 9c1df21b690..ba6bd38ca80 100644 --- a/src/couch/include/couch_db.hrl +++ b/src/couch/include/couch_db.hrl @@ -53,6 +53,8 @@ -define(INTERACTIVE_EDIT, interactive_edit). -define(REPLICATED_CHANGES, replicated_changes). +-define(LOG_UNEXPECTED_MSG(Msg), couch_log:warning("[~p:~p:~p/~p]{~p[~p]} Unexpected message: ~w", [?MODULE, ?LINE, ?FUNCTION_NAME, ?FUNCTION_ARITY, self(), element(2, process_info(self(), message_queue_len)), Msg])). + -type branch() :: {Key::term(), Value::term(), Tree::term()}. -type path() :: {Start::pos_integer(), branch()}. -type update_type() :: replicated_changes | interactive_edit. diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 6a7120f87ef..586c6e66a7a 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -306,6 +306,10 @@ {type, counter}, {desc, <<"number of couch_server LRU operations skipped">>} ]}. +{[couchdb, couch_server, open], [ + {type, counter}, + {desc, <<"number of couch_server open operations invoked">>} +]}. {[couchdb, query_server, vdu_rejects], [ {type, counter}, {desc, <<"number of rejections by validate_doc_update function">>} @@ -418,10 +422,39 @@ {type, counter}, {desc, <<"number of other requests">>} ]}. +{[couchdb, query_server, volume, ddoc_filter], [ + {type, counter}, + {desc, <<"number of docs filtered by ddoc filters">>} +]}. {[couchdb, legacy_checksums], [ {type, counter}, {desc, <<"number of legacy checksums found in couch_file instances">>} ]}. +{[couchdb, btree, folds], [ + {type, counter}, + {desc, <<"number of couch btree kv fold callback invocations">>} +]}. +{[couchdb, btree, get_node, kp_node], [ + {type, counter}, + {desc, <<"number of couch btree kp_nodes read">>} +]}. +{[couchdb, btree, get_node, kv_node], [ + {type, counter}, + {desc, <<"number of couch btree kv_nodes read">>} +]}. +{[couchdb, btree, write_node, kp_node], [ + {type, counter}, + {desc, <<"number of couch btree kp_nodes written">>} +]}. +{[couchdb, btree, write_node, kv_node], [ + {type, counter}, + {desc, <<"number of couch btree kv_nodes written">>} +]}. +%% CSRT (couch_stats_resource_tracker) stats +{[couchdb, csrt, delta_missing_t0], [ + {type, counter}, + {desc, <<"number of csrt contexts without a proper startime">>} +]}. {[pread, exceed_eof], [ {type, counter}, {desc, <<"number of the attempts to read beyond end of db file">>} diff --git a/src/couch/src/couch_btree.erl b/src/couch/src/couch_btree.erl index b974a22eeca..ba176cca2ca 100644 --- a/src/couch/src/couch_btree.erl +++ b/src/couch/src/couch_btree.erl @@ -472,6 +472,8 @@ reduce_tree_size(kp_node, NodeSize, [{_K, {_P, _Red, Sz}} | NodeList]) -> get_node(#btree{fd = Fd}, NodePos) -> {ok, {NodeType, NodeList}} = couch_file:pread_term(Fd, NodePos), + %% TODO: wire in csrt tracking + couch_stats:increment_counter([couchdb, btree, get_node, NodeType]), {NodeType, NodeList}. write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) -> @@ -480,6 +482,7 @@ write_node(#btree{fd = Fd, compression = Comp} = Bt, NodeType, NodeList) -> % now write out each chunk and return the KeyPointer pairs for those nodes ToWrite = [{NodeType, Chunk} || Chunk <- Chunks], WriteOpts = [{compression, Comp}], + couch_stats:increment_counter([couchdb, btree, write_node, NodeType]), {ok, PtrSizes} = couch_file:append_terms(Fd, ToWrite, WriteOpts), {ok, group_kps(Bt, NodeType, Chunks, PtrSizes)}. diff --git a/src/couch/src/couch_db.erl b/src/couch/src/couch_db.erl index e33e695c02e..694e829364f 100644 --- a/src/couch/src/couch_db.erl +++ b/src/couch/src/couch_db.erl @@ -298,6 +298,7 @@ open_doc(Db, IdOrDocInfo) -> open_doc(Db, IdOrDocInfo, []). open_doc(Db, Id, Options) -> + %% TODO: wire in csrt tracking increment_stat(Db, [couchdb, database_reads]), case open_doc_int(Db, Id, Options) of {ok, #doc{deleted = true} = Doc} -> @@ -1987,6 +1988,7 @@ increment_stat(#db{options = Options}, Stat, Count) when -> case lists:member(sys_db, Options) of true -> + %% TODO: we shouldn't leak resource usage just because it's a sys_db ok; false -> couch_stats:increment_counter(Stat, Count) diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 59ceeca13a1..003c3dc519d 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -246,8 +246,9 @@ bump_cmd_time_stat(Cmd, USec) when is_list(Cmd), is_integer(USec) -> bump_time_stat(ddoc_new, USec); [<<"ddoc">>, _, [<<"validate_doc_update">> | _] | _] -> bump_time_stat(ddoc_vdu, USec); - [<<"ddoc">>, _, [<<"filters">> | _] | _] -> - bump_time_stat(ddoc_filter, USec); + [<<"ddoc">>, _, [<<"filters">> | _], [Docs | _] | _] -> + bump_time_stat(ddoc_filter, USec), + bump_volume_stat(ddoc_filter, Docs); [<<"ddoc">> | _] -> bump_time_stat(ddoc_other, USec); _ -> @@ -258,6 +259,12 @@ bump_time_stat(Stat, USec) when is_atom(Stat), is_integer(USec) -> couch_stats:increment_counter([couchdb, query_server, calls, Stat]), couch_stats:increment_counter([couchdb, query_server, time, Stat], USec). +bump_volume_stat(ddoc_filter=Stat, Docs) when is_atom(Stat), is_list(Docs) -> + couch_stats:increment_counter([couchdb, query_server, volume, Stat, length(Docs)]); +bump_volume_stat(_, _) -> + %% TODO: handle other stats? + ok. + log_level("debug") -> debug; log_level("info") -> diff --git a/src/couch/src/couch_query_servers.erl b/src/couch/src/couch_query_servers.erl index 6789bfaef05..2d5185806ee 100644 --- a/src/couch/src/couch_query_servers.erl +++ b/src/couch/src/couch_query_servers.erl @@ -542,6 +542,8 @@ filter_docs(Req, Db, DDoc, FName, Docs) -> {ok, filter_docs_int(Db, DDoc, FName, JsonReq, JsonDocs)} catch throw:{os_process_error, {exit_status, 1}} -> + %% TODO: wire in csrt tracking + couch_stats:increment_counter([couchdb, query_server, js_filter_error]), %% batch used too much memory, retry sequentially. Fun = fun(JsonDoc) -> filter_docs_int(Db, DDoc, FName, JsonReq, [JsonDoc]) @@ -550,6 +552,12 @@ filter_docs(Req, Db, DDoc, FName, Docs) -> end. filter_docs_int(Db, DDoc, FName, JsonReq, JsonDocs) -> + %% Count usage in _int version as this can be repeated for OS error + %% Pros & cons... might not have actually processed `length(JsonDocs)` docs + %% but it certainly undercounts if we count in `filter_docs/5` above + %% TODO: replace with couchdb.query_server.*.ddoc_filter stats once we can + %% funnel back the stats used in the couchjs process to this caller process + csrt:js_filtered(length(JsonDocs)), [true, Passes] = ddoc_prompt( Db, DDoc, diff --git a/src/couch/src/couch_server.erl b/src/couch/src/couch_server.erl index ca12a56fa93..42ba80b1ea9 100644 --- a/src/couch/src/couch_server.erl +++ b/src/couch/src/couch_server.erl @@ -114,6 +114,8 @@ sup_start_link(N) -> gen_server:start_link({local, couch_server(N)}, couch_server, [N], []). open(DbName, Options) -> + %% TODO: wire in csrt tracking + couch_stats:increment_counter([couchdb, couch_server, open]), try validate_open_or_create(DbName, Options), open_int(DbName, Options) diff --git a/src/couch_stats/CSRT.md b/src/couch_stats/CSRT.md new file mode 100644 index 00000000000..073e89e09ac --- /dev/null +++ b/src/couch_stats/CSRT.md @@ -0,0 +1 @@ +# Couch Stats Resource Tracker (CSRT) diff --git a/src/couch_stats/src/couch_stats.app.src b/src/couch_stats/src/couch_stats.app.src index a54fac7349f..da100e06822 100644 --- a/src/couch_stats/src/couch_stats.app.src +++ b/src/couch_stats/src/couch_stats.app.src @@ -13,8 +13,13 @@ {application, couch_stats, [ {description, "Simple statistics collection"}, {vsn, git}, - {registered, [couch_stats_aggregator, couch_stats_process_tracker]}, - {applications, [kernel, stdlib]}, + {registered, [ + couch_stats_aggregator, + couch_stats_process_tracker, + csrt_server, + csrt_logger + ]}, + {applications, [kernel, stdlib, couch_log]}, {mod, {couch_stats_app, []}}, {env, []} ]}. diff --git a/src/couch_stats/src/couch_stats.erl b/src/couch_stats/src/couch_stats.erl index 29a4024491f..f92950fa940 100644 --- a/src/couch_stats/src/couch_stats.erl +++ b/src/couch_stats/src/couch_stats.erl @@ -24,6 +24,11 @@ update_gauge/2 ]). +%% couch_stats_resource_tracker API +-export([ + maybe_track_rexi_init_p/1 +]). + -type response() :: ok | {error, unknown_metric} | {error, invalid_metric}. -type stat() :: {any(), [{atom(), any()}]}. @@ -49,6 +54,11 @@ increment_counter(Name) -> -spec increment_counter(any(), pos_integer()) -> response(). increment_counter(Name, Value) -> + %% Should maybe_track_local happen before or after notify? + %% If after, only currently tracked metrics declared in the app's + %% stats_description.cfg will be trackable locally. Pros/cons. + %io:format("NOTIFY_EXISTING_METRIC: ~p || ~p || ~p~n", [Name, Op, Type]), + ok = maybe_track_local_counter(Name, Value), case couch_stats_util:get_counter(Name, stats()) of {ok, Ctx} -> couch_stats_counter:increment(Ctx, Value); {error, Error} -> {error, Error} @@ -100,6 +110,22 @@ stats() -> now_sec() -> erlang:monotonic_time(second). +%% Only potentially track positive increments to counters +-spec maybe_track_local_counter(any(), any()) -> ok. +maybe_track_local_counter(Name, Val) when is_integer(Val) andalso Val > 0 -> + %%io:format("maybe_track_local[~p]: ~p~n", [Val, Name]), + csrt:maybe_inc(Name, Val), + ok; +maybe_track_local_counter(_, _) -> + ok. + +maybe_track_rexi_init_p({M, F, _A}) -> + Metric = [M, F, spawned], + case csrt:should_track_init_p(Metric) of + true -> increment_counter(Metric); + false -> ok + end. + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl new file mode 100644 index 00000000000..1fd1a99a141 --- /dev/null +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -0,0 +1,171 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-define(CSRT, "csrt"). +-define(CSRT_INIT_P, "csrt.init_p"). + +%% CSRT pdict markers +-define(DELTA_TA, csrt_delta_ta). +-define(DELTA_TZ, csrt_delta_tz). %% T Zed instead of T0 +-define(PID_REF, csrt_pid_ref). %% track local ID +-define(TRACKER_PID, csrt_tracker). %% tracker pid + +-define(MANGO_EVAL_MATCH, mango_eval_match). +-define(DB_OPEN_DOC, docs_read). +-define(DB_OPEN, db_open). +-define(COUCH_SERVER_OPEN, db_open). +-define(COUCH_BT_GET_KP_NODE, get_kp_node). +-define(COUCH_BT_GET_KV_NODE, get_kv_node). +-define(COUCH_BT_WRITE_KP_NODE, write_kp_node). +-define(COUCH_BT_WRITE_KV_NODE, write_kv_node). +-define(COUCH_JS_FILTER, js_filter). +-define(COUCH_JS_FILTERED_DOCS, js_filtered_docs). +-define(IOQ_CALLS, ioq_calls). +-define(DOCS_WRITTEN, docs_written). +-define(ROWS_READ, rows_read). + +%% TODO: overlap between this and couch btree fold invocations +%% TODO: need some way to distinguish fols on views vs find vs all_docs +-define(FRPC_CHANGES_ROW, changes_processed). +-define(FRPC_CHANGES_RETURNED, changes_returned). + +-define(STATS_TO_KEYS, #{ + [mango, evaluate_selector] => ?MANGO_EVAL_MATCH, + [couchdb, database_reads] => ?DB_OPEN_DOC, + [fabric_rpc, changes, processed] => ?FRPC_CHANGES_ROW, + [fabric_rpc, changes, returned] => ?FRPC_CHANGES_RETURNED, + [fabric_rpc, view, rows_read] => ?ROWS_READ, + [couchdb, couch_server, open] => ?DB_OPEN, + [couchdb, btree, get_node, kp_node] => ?COUCH_BT_GET_KP_NODE, + [couchdb, btree, get_node, kv_node] => ?COUCH_BT_GET_KV_NODE, + [couchdb, btree, write_node, kp_node] => ?COUCH_BT_WRITE_KP_NODE, + [couchdb, btree, write_node, kv_node] => ?COUCH_BT_WRITE_KV_NODE, + %% NOTE: these stats are not local to the RPC worker, need forwarding + [couchdb, query_server, calls, ddoc_filter] => ?COUCH_JS_FILTER, + [couchdb, query_server, volume, ddoc_filter] => ?COUCH_JS_FILTERED_DOCS +}). + +-define(KEYS_TO_FIELDS, #{ + ?DB_OPEN => #rctx.?DB_OPEN, + ?ROWS_READ => #rctx.?ROWS_READ, + ?FRPC_CHANGES_RETURNED => #rctx.?FRPC_CHANGES_RETURNED, + ?DOCS_WRITTEN => #rctx.?DOCS_WRITTEN, + ?IOQ_CALLS => #rctx.?IOQ_CALLS, + ?COUCH_JS_FILTER => #rctx.?COUCH_JS_FILTER, + ?COUCH_JS_FILTERED_DOCS => #rctx.?COUCH_JS_FILTERED_DOCS, + ?MANGO_EVAL_MATCH => #rctx.?MANGO_EVAL_MATCH, + ?DB_OPEN_DOC => #rctx.?DB_OPEN_DOC, + ?FRPC_CHANGES_ROW => #rctx.?ROWS_READ, %% TODO: rework double use of rows_read + ?COUCH_BT_GET_KP_NODE => #rctx.?COUCH_BT_GET_KP_NODE, + ?COUCH_BT_GET_KV_NODE => #rctx.?COUCH_BT_GET_KV_NODE, + ?COUCH_BT_WRITE_KP_NODE => #rctx.?COUCH_BT_WRITE_KP_NODE, + ?COUCH_BT_WRITE_KV_NODE => #rctx.?COUCH_BT_WRITE_KV_NODE +}). + +-type pid_ref() :: {pid(), reference()}. +-type maybe_pid_ref() :: pid_ref() | undefined. +-type maybe_pid() :: pid() | undefined. + +-record(rpc_worker, { + mod :: atom() | '_', + func :: atom() | '_', + from :: pid_ref() | '_' +}). + +-record(coordinator, { + mod :: atom() | '_', + func :: atom() | '_', + method :: atom() | '_', + path :: binary() | '_' +}). + +-type coordinator() :: #coordinator{}. +-type rpc_worker() :: #rpc_worker{}. +-type rctx_type() :: coordinator() | rpc_worker(). + +-record(rctx, { + %% Metadata + started_at = csrt_util:tnow(), + updated_at = csrt_util:tnow(), + pid_ref :: maybe_pid_ref() | {'_', '_'}, + nonce, + type :: rctx_type() | undefined | '_', + dbname, + username, + + %% Stats counters + db_open = 0, + docs_read = 0 :: non_neg_integer(), + docs_written = 0 :: non_neg_integer(), + rows_read = 0 :: non_neg_integer(), + changes_processed = 0 :: non_neg_integer(), + changes_returned = 0 :: non_neg_integer(), + ioq_calls = 0 :: non_neg_integer(), + io_bytes_read = 0 :: non_neg_integer(), + io_bytes_written = 0 :: non_neg_integer(), + js_evals = 0 :: non_neg_integer(), + js_filter = 0 :: non_neg_integer(), + js_filtered_docs = 0 :: non_neg_integer(), + mango_eval_match = 0 :: non_neg_integer(), + %% TODO: switch record definitions to be macro based, eg: + %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer(), + get_kv_node = 0 :: non_neg_integer(), + get_kp_node = 0 :: non_neg_integer(), + write_kv_node = 0 :: non_neg_integer(), + write_kp_node = 0 :: non_neg_integer() +}). + +-type rctx_field() :: + started_at + | updated_at + | pid_ref + | nonce + | type + | dbname + | username + | db_open + | docs_read + | docs_written + | rows_read + | changes_processed + | changes_returned + | ioq_calls + | io_bytes_read + | io_bytes_written + | js_evals + | js_filter + | js_filtered_docs + | mango_eval_match + | get_kv_node + | get_kp_node + | write_kv_node + | write_kp_node. + +-type coordinator_rctx() :: #rctx{type :: coordinator()}. +-type rpc_worker_rctx() :: #rctx{type :: rpc_worker()}. +-type rctx() :: #rctx{} | coordinator_rctx() | rpc_worker_rctx(). +-type maybe_rctx() :: rctx() | undefined. + +%% TODO: solidify nonce type +-type nonce() :: any(). +-type dbname() :: iodata(). +-type username() :: iodata(). + +-type delta() :: map(). +-type maybe_delta() :: delta() | undefined. +-type tagged_delta() :: {delta, maybe_delta()}. + +-type matcher_name() :: string(). %% TODO: switch to string to allow dynamic options +-type matcher() :: {ets:match_spec(), ets:comp_match_spec()}. +-type maybe_matcher() :: matcher() | undefined. +-type matchers() :: #{matcher_name() => matcher()}. +-type maybe_matchers() :: matchers() | undefined. diff --git a/src/couch_stats/src/couch_stats_sup.erl b/src/couch_stats/src/couch_stats_sup.erl index 325372c3e4b..826dfbac4fc 100644 --- a/src/couch_stats/src/couch_stats_sup.erl +++ b/src/couch_stats/src/couch_stats_sup.erl @@ -29,6 +29,8 @@ init([]) -> { {one_for_one, 5, 10}, [ ?CHILD(couch_stats_server, worker), + ?CHILD(csrt_server, worker), + ?CHILD(csrt_logger, worker), ?CHILD(couch_stats_process_tracker, worker) ] }}. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl new file mode 100644 index 00000000000..a2b5f51d85c --- /dev/null +++ b/src/couch_stats/src/csrt.erl @@ -0,0 +1,500 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch_stats_resource_tracker.hrl"). + +%% PidRef API +-export([ + destroy_pid_ref/0, + destroy_pid_ref/1, + create_pid_ref/0, + get_pid_ref/0, + get_pid_ref/1, + set_pid_ref/1 +]). + +%% Context API +-export([ + create_context/2, + create_coordinator_context/2, + create_worker_context/3, + destroy_context/0, + destroy_context/1, + get_resource/0, + get_resource/1, + set_context_dbname/1, + set_context_dbname/2, + set_context_handler_fun/1, + set_context_handler_fun/2, + set_context_username/1, + set_context_username/2 +]). + +%% Public API +-export([ + is_enabled/0, + is_enabled_init_p/0, + do_report/2, + maybe_report/2, + conf_get/1, + conf_get/2 +]). + +%% stats collection api +-export([ + accumulate_delta/1, + add_delta/2, + docs_written/1, + extract_delta/1, + get_delta/0, + inc/1, + inc/2, + ioq_called/0, + js_filtered/1, + make_delta/0, + rctx_delta/2, + maybe_add_delta/1, + maybe_add_delta/2, + maybe_inc/2, + should_track_init_p/1 +]). + +%% aggregate query api +-export([ + active/0, + active/1, + active_coordinators/0, + active_coordinators/1, + active_workers/0, + active_workers/1, + count_by/1, + find_by_nonce/1, + find_by_pid/1, + find_by_pidref/1, + find_workers_by_pidref/1, + group_by/2, + group_by/3, + sorted/1, + sorted_by/1, + sorted_by/2, + sorted_by/3 +]). + +%% +%% PidRef operations +%% + +-spec get_pid_ref() -> maybe_pid_ref(). +get_pid_ref() -> + csrt_util:get_pid_ref(). + +-spec get_pid_ref(Rctx :: rctx()) -> pid_ref(). +get_pid_ref(Rctx) -> + csrt_util:get_pid_ref(Rctx). + +-spec set_pid_ref(PidRef :: pid_ref()) -> pid_ref(). +set_pid_ref(PidRef) -> + csrt_util:set_pid_ref(PidRef). + +-spec create_pid_ref() -> pid_ref(). +create_pid_ref() -> + csrt_server:create_pid_ref(). + +-spec destroy_pid_ref() -> maybe_pid_ref(). +destroy_pid_ref() -> + destroy_pid_ref(get_pid_ref()). + +%%destroy_pid_ref(undefined) -> +%% undefined; +-spec destroy_pid_ref(PidRef :: maybe_pid_ref()) -> maybe_pid_ref(). +destroy_pid_ref(_PidRef) -> + erase(?PID_REF). + +%% +%% Context lifecycle API +%% + +%% TODO: shouldn't need this? +%% create_resource(#rctx{} = Rctx) -> +%% csrt_server:create_resource(Rctx). + +-spec create_worker_context(From, MFA, Nonce) -> pid_ref() | false when + From :: pid_ref(), MFA :: mfa(), Nonce :: term(). +create_worker_context(From, {M,F,_A}, Nonce) -> + case is_enabled() of + true -> + Type = #rpc_worker{from=From, mod=M, func=F}, + create_context(Type, Nonce); + false -> + false + end. + +-spec create_coordinator_context(Httpd , Path) -> pid_ref() | false when + Httpd :: #httpd{}, Path :: list(). +create_coordinator_context(#httpd{method=Verb, nonce=Nonce}, Path0) -> + case is_enabled() of + true -> + Path = list_to_binary([$/ | Path0]), + Type = #coordinator{method=Verb, path=Path}, + create_context(Type, Nonce); + false -> + false + end. + +-spec create_context(Type :: rctx_type(), Nonce :: term()) -> pid_ref(). +create_context(Type, Nonce) -> + Rctx = csrt_server:new_context(Type, Nonce), + %% TODO: which approach + %% PidRef = csrt_server:pid_ref(Rctx), + PidRef = get_pid_ref(Rctx), + set_pid_ref(PidRef), + csrt_util:set_delta_zero(Rctx), + csrt_util:set_delta_a(Rctx), + csrt_server:create_resource(Rctx), + csrt_logger:track(Rctx), + PidRef. + +-spec set_context_dbname(DbName :: binary()) -> boolean(). +set_context_dbname(DbName) -> + set_context_dbname(DbName, get_pid_ref()). + +-spec set_context_dbname(DbName, PidRef) -> boolean() when + DbName :: binary(), PidRef :: pid_ref() | undefined. +set_context_dbname(_, undefined) -> + false; +set_context_dbname(DbName, PidRef) -> + is_enabled() andalso csrt_server:set_context_dbname(DbName, PidRef). + +-spec set_context_handler_fun(Fun :: function()) -> boolean(). +set_context_handler_fun(Fun) when is_function(Fun) -> + case is_enabled() of + false -> + false; + true -> + FProps = erlang:fun_info(Fun), + Mod = proplists:get_value(module, FProps), + Func = proplists:get_value(name, FProps), + update_handler_fun(Mod, Func, get_pid_ref()) + end. + +-spec set_context_handler_fun(Mod :: atom(), Func :: atom()) -> boolean(). +set_context_handler_fun(Mod, Func) + when is_atom(Mod) andalso is_atom(Func) -> + case is_enabled() of + false -> + false; + true -> + update_handler_fun(Mod, Func, get_pid_ref()) + end. + +-spec update_handler_fun(Mod, Func, PidRef) -> boolean() when + Mod :: atom(), Func :: atom(), PidRef :: maybe_pid_ref(). +update_handler_fun(_, _, undefined) -> + false; +update_handler_fun(Mod, Func, PidRef) -> + Rctx = get_resource(PidRef), + %% TODO: #coordinator{} assumption needs to adapt for other types + #coordinator{} = Coordinator0 = csrt_server:get_context_type(Rctx), + Coordinator = Coordinator0#coordinator{mod=Mod, func=Func}, + csrt_server:set_context_type(Coordinator, PidRef), + ok. + +%% @equiv set_context_username(User, get_pid_ref()) +set_context_username(User) -> + set_context_username(User, get_pid_ref()). + +-spec set_context_username(User, PidRef) -> boolean() when + User :: null | undefined | #httpd{} | #user_ctx{} | binary(), + PidRef :: maybe_pid_ref(). +set_context_username(null, _) -> + false; +set_context_username(_, undefined) -> + false; +set_context_username(#httpd{user_ctx = Ctx}, PidRef) -> + set_context_username(Ctx, PidRef); +set_context_username(#user_ctx{name = Name}, PidRef) -> + set_context_username(Name, PidRef); +set_context_username(UserName, PidRef) -> + is_enabled() andalso csrt_server:set_context_username(UserName, PidRef). + +-spec destroy_context() -> ok. +destroy_context() -> + destroy_context(get_pid_ref()). + +-spec destroy_context(PidRef :: maybe_pid_ref()) -> ok. +destroy_context(undefined) -> + ok; +destroy_context({_, _} = PidRef) -> + csrt_logger:stop_tracker(), + destroy_pid_ref(PidRef), + ok. + +%% +%% Public API +%% + +%% @equiv csrt_util:is_enabled(). +-spec is_enabled() -> boolean(). +is_enabled() -> + csrt_util:is_enabled(). + +%% @equiv csrt_util:is_enabled_init_p(). +-spec is_enabled_init_p() -> boolean(). +is_enabled_init_p() -> + csrt_util:is_enabled_init_p(). + +-spec get_resource() -> maybe_rctx(). +get_resource() -> + get_resource(get_pid_ref()). + +-spec get_resource(PidRef :: maybe_pid_ref()) -> maybe_rctx(). +get_resource(PidRef) -> + csrt_server:get_resource(PidRef). + +%% Log a CSRT report if any filters match +-spec maybe_report(ReportName :: string(), PidRef :: pid_ref()) -> ok. +maybe_report(ReportName, PidRef) -> + csrt_logger:maybe_report(ReportName, PidRef). + +%% Direct report logic skipping should log filters +-spec do_report(ReportName :: string(), PidRef :: pid_ref()) -> boolean(). +do_report(ReportName, PidRef) -> + csrt_logger:do_report(ReportName, get_resource(PidRef)). + +%% +%% Stat collection API +%% + +-spec inc(Key :: rctx_field()) -> non_neg_integer(). +inc(Key) -> + is_enabled() andalso csrt_server:inc(get_pid_ref(), Key). + +-spec inc(Key :: rctx_field(), N :: non_neg_integer()) -> non_neg_integer(). +inc(Key, N) when is_integer(N) andalso N >= 0 -> + is_enabled() andalso csrt_server:inc(get_pid_ref(), Key, N). + + +-spec maybe_inc(Stat :: atom(), Val :: non_neg_integer()) -> non_neg_integer(). +maybe_inc(Stat, Val) -> + case maps:is_key(Stat, ?STATS_TO_KEYS) of + true -> + inc(maps:get(Stat, ?STATS_TO_KEYS), Val); + false -> + 0 + end. + +-spec should_track_init_p(Stat :: [atom()]) -> boolean(). +should_track_init_p([Mod, Func, spawned]) -> + is_enabled_init_p() andalso csrt_util:should_track_init_p(Mod, Func); +should_track_init_p(_Metric) -> + false. + +-spec ioq_called() -> non_neg_integer(). +ioq_called() -> + inc(ioq_calls). + +%% we cannot yet use stats couchdb.query_server.*.ddoc_filter because those +%% are collected in a dedicated process. +%% TODO: funnel back stats from background worker processes to the RPC worker +js_filtered(N) -> + inc(js_filter), + inc(js_filtered_docs, N). + +docs_written(N) -> + inc(docs_written, N). + +-spec accumulate_delta(Delta :: map() | undefined) -> ok. +accumulate_delta(Delta) when is_map(Delta) -> + %% TODO: switch to creating a batch of updates to invoke a single + %% update_counter rather than sequentially invoking it for each field + is_enabled() andalso maps:foreach(fun inc/2, Delta), + ok; +accumulate_delta(undefined) -> + ok. + +-spec make_delta() -> maybe_delta(). +make_delta() -> + case is_enabled() of + false -> + undefined; + true -> + csrt_util:make_delta(get_pid_ref()) + end. + +-spec rctx_delta(TA :: maybe_rctx(), TB :: maybe_rctx()) -> maybe_delta(). +rctx_delta(TA, TB) -> + csrt_util:rctx_delta(TA, TB). + +%% TODO: cleanup return type +%%-spec update_counter(Field :: rctx_field(), Count :: non_neg_integer()) -> false | ok. +%%-spec update_counter(Field :: non_neg_integer(), Count :: non_neg_integer()) -> false | ok. +%%update_counter(_Field, Count) when Count < 0 -> +%% false; +%%update_counter(Field, Count) when Count >= 0 -> +%% is_enabled() andalso csrt_server:update_counter(get_pid_ref(), Field, Count). + + +-spec conf_get(Key :: string()) -> string(). +conf_get(Key) -> + csrt_util:conf_get(Key). + + +-spec conf_get(Key :: string(), Default :: string()) -> string(). +conf_get(Key, Default) -> + csrt_util:conf_get(Key, Default). + +%% +%% aggregate query api +%% + +-spec active() -> [rctx()]. +active() -> + csrt_query:active(). + +%% TODO: ensure Type fields align with type specs +%%-spec active(Type :: rctx_type()) -> [rctx()]. +-spec active(Type :: json) -> [rctx()]. +active(Type) -> + csrt_query:active(Type). + +-spec active_coordinators() -> [coordinator_rctx()]. +active_coordinators() -> + csrt_query:active_coordinators(). + +%% TODO: cleanup json logic here +-spec active_coordinators(Type :: json) -> [coordinator_rctx()]. +active_coordinators(Type) -> + csrt_query:active_coordinators(Type). + +-spec active_workers() -> [rpc_worker_rctx()]. +active_workers() -> + csrt_query:active_workers(). + +-spec active_workers(Type :: json) -> [rpc_worker_rctx()]. +active_workers(Type) -> + csrt_query:active_workers(Type). + +-spec count_by(Key :: string()) -> map(). +count_by(Key) -> + csrt_query:count_by(Key). + +find_by_nonce(Nonce) -> + csrt_query:find_by_nonce(Nonce). + +find_by_pid(Pid) -> + csrt_query:find_by_pid(Pid). + +find_by_pidref(PidRef) -> + csrt_query:find_by_pidref(PidRef). + +find_workers_by_pidref(PidRef) -> + csrt_query:find_workers_by_pidref(PidRef). + +group_by(Key, Val) -> + csrt_query:group_by(Key, Val). + +group_by(Key, Val, Agg) -> + csrt_query:group_by(Key, Val, Agg). + +sorted(Map) -> + csrt_query:sorted(Map). + +sorted_by(Key) -> + csrt_query:sorted_by(Key). + +sorted_by(Key, Val) -> + csrt_query:sorted_by(Key, Val). + +sorted_by(Key, Val, Agg) -> + csrt_query:sorted_by(Key, Val, Agg). + +%% +%% Delta API +%% + +add_delta(T, Delta) -> + csrt_util:add_delta(T, Delta). + +extract_delta(T) -> + csrt_util:extract_delta(T). + + +get_delta() -> + csrt_util:get_delta(get_pid_ref()). + +maybe_add_delta(T) -> + csrt_util:maybe_add_delta(T). + +maybe_add_delta(T, Delta) -> + csrt_util:maybe_add_delta(T, Delta). + +%% +%% Internal Operations assuming is_enabled() == true +%% + + +-ifdef(TEST). + +-include_lib("couch/include/couch_eunit.hrl"). + +couch_stats_resource_tracker_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_static_map_translations), + ?TDEF_FE(t_should_track_init_p), + ?TDEF_FE(t_should_not_track_init_p) + ] + }. + +setup() -> + test_util:start_couch(). + +teardown(Ctx) -> + test_util:stop_couch(Ctx). + +t_static_map_translations(_) -> + ?assert(lists:all(fun(E) -> maps:is_key(E, ?KEYS_TO_FIELDS) end, maps:values(?STATS_TO_KEYS))), + %% TODO: properly handle ioq_calls field + ?assertEqual(lists:sort(maps:values(?STATS_TO_KEYS)), lists:delete(docs_written, lists:delete(ioq_calls, lists:sort(maps:keys(?KEYS_TO_FIELDS))))). + +t_should_track_init_p(_) -> + config:set(?CSRT_INIT_P, "enabled", "true", false), + Metrics = [ + [fabric_rpc, all_docs, spawned], + [fabric_rpc, changes, spawned], + [fabric_rpc, map_view, spawned], + [fabric_rpc, reduce_view, spawned], + [fabric_rpc, get_all_security, spawned], + [fabric_rpc, open_doc, spawned], + [fabric_rpc, update_docs, spawned], + [fabric_rpc, open_shard, spawned] + ], + [csrt_util:set_fabric_init_p(F, true, false) || [_, F, _] <- Metrics], + [?assert(should_track_init_p(M), M) || M <- Metrics]. + +t_should_not_track_init_p(_) -> + config:set(?CSRT_INIT_P, "enabled", "true", false), + Metrics = [ + [couch_db, name, spawned], + [couch_db, get_db_info, spawned], + [couch_db, open, spawned], + [fabric_rpc, get_purge_seq, spawned] + ], + [?assert(should_track_init_p(M) =:= false, M) || M <- Metrics]. + +-endif. diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl new file mode 100644 index 00000000000..2e692e88836 --- /dev/null +++ b/src/couch_stats/src/csrt_logger.erl @@ -0,0 +1,401 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_logger). + +%% Process lifetime logging api +-export([ + get_tracker/0, + log_process_lifetime_report/1, + put_tracker/1, + stop_tracker/0, + stop_tracker/1, + track/1, + tracker/1 +]). + +%% Raw API that bypasses is_enabled checks +-export([ + do_lifetime_report/1, + do_status_report/1, + do_report/2, + maybe_report/2 +]). + +-export([ + start_link/0, + init/1, + handle_call/3, + handle_cast/2, + handle_info/2 +]). + +%% Config update subscription API +-export([ + subscribe_changes/0, + handle_config_change/5, + handle_config_terminate/3 +]). + +%% Matchers +-export([ + get_matcher/1, + get_matchers/0, + is_match/1, + is_match/2, + matcher_on_dbname/1, + matcher_on_docs_read/1, + matcher_on_docs_written/1, + matcher_on_rows_read/1, + matcher_on_worker_changes_processed/1, + matcher_on_ioq_calls/1, + matcher_on_nonce/1, + reload_matchers/0 +]). + +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("couch_stats_resource_tracker.hrl"). + +-define(MATCHERS_KEY, {?MODULE, all_csrt_matchers}). +-define(CONF_MATCHERS_ENABLED, "csrt_logger.matchers_enabled"). +-define(CONF_MATCHERS_THRESHOLD, "csrt_logger.matchers_threshold"). +-define(CONF_MATCHERS_DBNAMES, "csrt_logger.dbnames_io"). + +-record(st, { + matchers = #{} +}). + +-spec track(Rctx :: rctx()) -> pid(). +track(#rctx{pid_ref=PidRef}) -> + case get_tracker() of + undefined -> + Pid = spawn(?MODULE, tracker, [PidRef]), + put_tracker(Pid), + Pid; + Pid when is_pid(Pid) -> + Pid + end. + +-spec tracker(PidRef :: pid_ref()) -> ok. +tracker({Pid, _Ref}=PidRef) -> + MonRef = erlang:monitor(process, Pid), + receive + stop -> + %% TODO: do we need cleanup here? + log_process_lifetime_report(PidRef), + csrt_server:destroy_resource(PidRef), + ok; + {'DOWN', MonRef, _Type, _0DPid, _Reason0} -> + %% TODO: should we pass reason to log_process_lifetime_report? + %% Reason = case Reason0 of + %% {shutdown, Shutdown0} -> + %% Shutdown = atom_to_binary(Shutdown0), + %% <<"shutdown: ", Shutdown/binary>>; + %% Reason0 -> + %% Reason0 + %% end, + %% TODO: should we send the induced work delta to the coordinator? + log_process_lifetime_report(PidRef), + csrt_server:destroy_resource(PidRef), + ok + end. + +-spec log_process_lifetime_report(PidRef :: pid_ref()) -> ok. +log_process_lifetime_report(PidRef) -> + case csrt_util:is_enabled() of + true -> + maybe_report("csrt-pid-usage-lifetime", PidRef); + false -> + ok + end. + +%% TODO: add Matchers spec +-spec find_matches(Rctxs :: [rctx()], Matchers :: [any()]) -> matchers(). +find_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> + maps:filter( + fun(_Name, {_MSpec, CompMSpec}) -> + catch [] =/= ets:match_spec_run(Rctxs, CompMSpec) + end, + Matchers + ). + +-spec reload_matchers() -> ok. +reload_matchers() -> + ok = gen_server:call(?MODULE, reload_matchers, infinity). + +-spec get_matchers() -> matchers(). +get_matchers() -> + persistent_term:get(?MATCHERS_KEY, #{}). + +-spec get_matcher(Name :: matcher_name()) -> maybe_matcher(). +get_matcher(Name) -> + maps:get(Name, get_matchers(), undefined). + +-spec is_match(Rctx :: maybe_rctx()) -> boolean(). +is_match(undefined) -> + false; +is_match(#rctx{}=Rctx) -> + is_match(Rctx, get_matchers()). + +%% TODO: add Matchers spec +-spec is_match(Rctx :: maybe_rctx(), Matchers :: [any()]) -> boolean(). +is_match(undefined, _Matchers) -> + false; +is_match(_Rctx, undefined) -> + false; +is_match(#rctx{}=Rctx, Matchers) when is_map(Matchers) -> + maps:size(find_matches([Rctx], Matchers)) > 0. + +-spec maybe_report(ReportName :: string(), PidRef :: maybe_pid_ref()) -> ok. +maybe_report(ReportName, PidRef) -> + Rctx = csrt_server:get_resource(PidRef), + case is_match(Rctx) of + true -> + do_report(ReportName, Rctx), + ok; + false -> + ok + end. + +-spec do_lifetime_report(Rctx :: rctx()) -> boolean(). +do_lifetime_report(Rctx) -> + do_report("csrt-pid-usage-lifetime", Rctx). + +-spec do_status_report(Rctx :: rctx()) -> boolean(). +do_status_report(Rctx) -> + do_report("csrt-pid-usage-status", Rctx). + +-spec do_report(ReportName :: string(), Rctx :: rctx()) -> boolean(). +do_report(ReportName, #rctx{}=Rctx) -> + couch_log:report(ReportName, csrt_util:to_json(Rctx)). + +%% +%% Process lifetime logging api +%% + +-spec get_tracker() -> maybe_pid(). +get_tracker() -> + get(?TRACKER_PID). + +-spec put_tracker(Pid :: pid()) -> maybe_pid(). +put_tracker(Pid) when is_pid(Pid) -> + put(?TRACKER_PID, Pid). + +-spec stop_tracker() -> ok. +stop_tracker() -> + stop_tracker(get_tracker()). + +-spec stop_tracker(Pid :: maybe_pid()) -> ok. +stop_tracker(undefined) -> + ok; +stop_tracker(Pid) when is_pid(Pid) -> + Pid ! stop, + ok. + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init([]) -> + ok = initialize_matchers(), + ok = subscribe_changes(), + {ok, #st{}}. + +handle_call({register, Name, MSpec}, _From, #st{matchers=Matchers}=St) -> + case add_matcher(Name, MSpec, Matchers) of + {ok, Matchers1} -> + set_matchers_term(Matchers1), + {reply, ok, St#st{matchers=Matchers1}}; + {error, badarg}=Error -> + {reply, Error, St} + end; +handle_call(reload_matchers, _From, St) -> + couch_log:warning("Reloading persistent term matchers", []), + ok = initialize_matchers(), + {reply, ok, St}; +handle_call(_, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State, 0}. + +handle_info(restart_config_listener, State) -> + ok = config:listen_for_changes(?MODULE, nil), + {noreply, State}; +handle_info(_Msg, St) -> + {noreply, St}. + +%% +%% Matchers +%% + +-spec matcher_on_dbname(DbName :: dbname()) -> ets:match_spec(). +matcher_on_dbname(DbName) + when is_binary(DbName) -> + ets:fun2ms(fun(#rctx{dbname=DbName1} = R) when DbName =:= DbName1 -> R end). + +-spec matcher_on_dbname_io_threshold(DbName, Threshold) -> ets:match_spec() when + DbName :: dbname(), Threshold :: pos_integer(). +matcher_on_dbname_io_threshold(DbName, Threshold) + when is_binary(DbName) -> + ets:fun2ms(fun(#rctx{dbname=DbName1, ioq_calls=IOQ, get_kv_node=KVN, get_kp_node=KPN, docs_read=Docs, rows_read=Rows, changes_processed=Chgs} = R) when DbName =:= DbName1 andalso ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or (Rows >= Threshold) or (Chgs >= Threshold)) -> R end). + +-spec matcher_on_docs_read(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_docs_read(Threshold) + when is_integer(Threshold) andalso Threshold > 0 -> + %%ets:fun2ms(fun(#rctx{type=#coordinator{}, docs_read=DocsRead} = R) when DocsRead >= Threshold -> R end). + ets:fun2ms(fun(#rctx{docs_read=DocsRead} = R) when DocsRead >= Threshold -> R end). + +-spec matcher_on_docs_written(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_docs_written(Threshold) + when is_integer(Threshold) andalso Threshold > 0 -> + %%ets:fun2ms(fun(#rctx{type=#coordinator{}, docs_written=DocsRead} = R) when DocsRead >= Threshold -> R end). + ets:fun2ms(fun(#rctx{docs_written=DocsWritten} = R) when DocsWritten >= Threshold -> R end). + +-spec matcher_on_rows_read(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_rows_read(Threshold) + when is_integer(Threshold) andalso Threshold > 0 -> + ets:fun2ms(fun(#rctx{rows_read=RowsRead} = R) when RowsRead >= Threshold -> R end). + +-spec matcher_on_nonce(Nonce :: nonce()) -> ets:match_spec(). +matcher_on_nonce(Nonce) -> + ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end). + +-spec matcher_on_worker_changes_processed(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_worker_changes_processed(Threshold) + when is_integer(Threshold) andalso Threshold > 0 -> + ets:fun2ms( + fun( + #rctx{ + changes_processed=Processed, + changes_returned=Returned + } = R + ) when (Processed - Returned) >= Threshold -> + R + end + ). + +-spec matcher_on_ioq_calls(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_ioq_calls(Threshold) + when is_integer(Threshold) andalso Threshold > 0 -> + ets:fun2ms(fun(#rctx{ioq_calls=IOQCalls} = R) when IOQCalls >= Threshold -> R end). + +-spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, badarg} when + Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). +add_matcher(Name, MSpec, Matchers) -> + try ets:match_spec_compile(MSpec) of + CompMSpec -> + %% TODO: handle already registered name case + Matchers1 = maps:put(Name, {MSpec, CompMSpec}, Matchers), + {ok, Matchers1} + catch + error:badarg -> + {error, badarg} + end. + +-spec set_matchers_term(Matchers :: matchers()) -> maybe_matchers(). +set_matchers_term(Matchers) when is_map(Matchers) -> + persistent_term:put({?MODULE, all_csrt_matchers}, Matchers). + +-spec initialize_matchers() -> ok. +initialize_matchers() -> + DefaultMatchers = [ + {docs_read, fun matcher_on_docs_read/1, 100}, + {dbname, fun matcher_on_dbname/1, <<"foo">>}, + {rows_read, fun matcher_on_rows_read/1, 100}, + {docs_written, fun matcher_on_docs_written/1, 1}, + %%{view_rows_read, fun matcher_on_rows_read/1, 1000}, + %%{slow_reqs, fun matcher_on_slow_reqs/1, 10000}, + {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, + {ioq_calls, fun matcher_on_ioq_calls/1, 10000} + ], + Matchers = lists:foldl( + fun({Name0, MatchGenFunc, Threshold0}, Matchers0) when is_atom(Name0) -> + Name = atom_to_list(Name0), + case matcher_enabled(Name) of + true -> + Threshold = matcher_threshold(Name, Threshold0), + %% TODO: handle errors from Func + case add_matcher(Name, MatchGenFunc(Threshold), Matchers0) of + {ok, Matchers1} -> + Matchers1; + {error, badarg} -> + couch_log:warning("[~p] Failed to initialize matcher: ~p", [?MODULE, Name]), + Matchers0 + end; + false -> + Matchers0 + end + end, + #{}, + DefaultMatchers + ), + Matchers1 = lists:foldl( + fun({Dbname, Value}, Matchers0) -> + try list_to_integer(Value) of + Threshold when Threshold > 0 -> + Name = "dbname_io__" ++ Dbname ++ "__" ++ Value, + DbnameB = list_to_binary(Dbname), + MSpec = matcher_on_dbname_io_threshold(DbnameB, Threshold), + case add_matcher(Name, MSpec, Matchers0) of + {ok, Matchers1} -> + Matchers1; + {error, badarg} -> + couch_log:warning("[~p] Failed to initialize matcher: ~p", [?MODULE, Name]), + Matchers0 + end; + _ -> + Matchers0 + catch error:badarg -> + couch_log:warning("[~p] Failed to initialize dbname io matcher on: ~p", [?MODULE, Dbname]) + end + end, + Matchers, + config:get(?CONF_MATCHERS_DBNAMES) + ), + + couch_log:notice("Initialized ~p CSRT Logger matchers", [maps:size(Matchers1)]), + persistent_term:put(?MATCHERS_KEY, Matchers1), + ok. + +-spec matcher_enabled(Name :: string()) -> boolean(). +matcher_enabled(Name) when is_list(Name) -> + %% TODO: fix + %% config:get_boolean(?CONF_MATCHERS_ENABLED, Name, false). + config:get_boolean(?CONF_MATCHERS_ENABLED, Name, true). + +-spec matcher_threshold(Name, Threshold) -> string() | integer() when + Name :: string(), Threshold :: pos_integer() | string(). +matcher_threshold("dbname", DbName) when is_binary(DbName) -> + %% TODO: toggle Default to undefined to disallow for particular dbname + %% TODO: sort out list vs binary + %%config:get_integer(?CONF_MATCHERS_THRESHOLD, binary_to_list(DbName), Default); + DbName; +matcher_threshold(Name, Default) + when is_list(Name) andalso is_integer(Default) andalso Default > 0 -> + config:get_integer(?CONF_MATCHERS_THRESHOLD, Name, Default). + +subscribe_changes() -> + config:listen_for_changes(?MODULE, nil). + +handle_config_change(?CONF_MATCHERS_ENABLED, _Key, _Val, _Persist, St) -> + ok = gen_server:call(?MODULE, reload_matchers, infinity), + {ok, St}; +handle_config_change(?CONF_MATCHERS_THRESHOLD, _Key, _Val, _Persist, St) -> + ok = gen_server:call(?MODULE, reload_matchers, infinity), + {ok, St}; +handle_config_change(_Sec, _Key, _Val, _Persist, St) -> + {ok, St}. + +handle_config_terminate(_, stop, _) -> + ok; +handle_config_terminate(_, _, _) -> + erlang:send_after(5000, whereis(?MODULE), restart_config_listener). diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl new file mode 100644 index 00000000000..a3580c58a8d --- /dev/null +++ b/src/couch_stats/src/csrt_query.erl @@ -0,0 +1,174 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_query). + +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("couch_stats_resource_tracker.hrl"). + +%% aggregate query api +-export([ + active/0, + active/1, + active_coordinators/0, + active_coordinators/1, + active_workers/0, + active_workers/1, + count_by/1, + find_by_nonce/1, + find_by_pid/1, + find_by_pidref/1, + find_workers_by_pidref/1, + group_by/2, + group_by/3, + sorted/1, + sorted_by/1, + sorted_by/2, + sorted_by/3 +]). + +%% +%% Aggregate query API +%% + +active() -> + active_int(all). + +active_coordinators() -> + active_int(coordinators). + +active_workers() -> + active_int(workers). + +%% active_json() or active(json)? +active(json) -> + to_json_list(active_int(all)). + +active_coordinators(json) -> + to_json_list(active_int(coordinators)). + +active_workers(json) -> + to_json_list(active_int(workers)). + +active_int(coordinators) -> + select_by_type(coordinators); +active_int(workers) -> + select_by_type(workers); +active_int(all) -> + select_by_type(all). + +select_by_type(coordinators) -> + ets:select(?MODULE, ets:fun2ms(fun(#rctx{type = #coordinator{}} = R) -> R end)); +select_by_type(workers) -> + ets:select(?MODULE, ets:fun2ms(fun(#rctx{type = #rpc_worker{}} = R) -> R end)); +select_by_type(all) -> + ets:tab2list(?MODULE). + +find_by_nonce(Nonce) -> + %%ets:match_object(?MODULE, ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end)). + [R || R <- ets:match_object(?MODULE, #rctx{nonce=Nonce})]. + +find_by_pid(Pid) -> + %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{pid_ref={Pid, '_'}, _ = '_'})]. + [R || R <- ets:match_object(?MODULE, #rctx{pid_ref={Pid, '_'}})]. + +find_by_pidref(PidRef) -> + %%[R || R <- ets:match_object(?MODULE, #rctx{pid_ref=PidRef, _ = '_'})]. + [R || R <- ets:match_object(?MODULE, #rctx{pid_ref=PidRef})]. + +find_workers_by_pidref(PidRef) -> + %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{type=#rpc_worker{from=PidRef}, _ = '_'})]. + [R || R <- ets:match_object(?MODULE, #rctx{type=#rpc_worker{from=PidRef}})]. + +field(#rctx{pid_ref=Val}, pid_ref) -> Val; +%% NOTE: Pros and cons to doing these convert functions here +%% Ideally, this would be done later so as to prefer the core data structures +%% as long as possible, but we currently need the output of this function to +%% be jiffy:encode'able. The tricky bit is dynamically encoding the group_by +%% structure provided by the caller of *_by aggregator functions below. +%% For now, we just always return jiffy:encode'able data types. +field(#rctx{nonce=Val}, nonce) -> Val; +%%field(#rctx{from=Val}, from) -> Val; +%% TODO: fix this, perhaps move it all to csrt_util? +field(#rctx{type=Val}, type) -> csrt_util:convert_type(Val); +field(#rctx{dbname=Val}, dbname) -> Val; +field(#rctx{username=Val}, username) -> Val; +%%field(#rctx{path=Val}, path) -> Val; +field(#rctx{db_open=Val}, db_open) -> Val; +field(#rctx{docs_read=Val}, docs_read) -> Val; +field(#rctx{rows_read=Val}, rows_read) -> Val; +field(#rctx{changes_processed=Val}, changes_processed) -> Val; +field(#rctx{changes_returned=Val}, changes_returned) -> Val; +field(#rctx{ioq_calls=Val}, ioq_calls) -> Val; +field(#rctx{io_bytes_read=Val}, io_bytes_read) -> Val; +field(#rctx{io_bytes_written=Val}, io_bytes_written) -> Val; +field(#rctx{js_evals=Val}, js_evals) -> Val; +field(#rctx{js_filter=Val}, js_filter) -> Val; +field(#rctx{js_filtered_docs=Val}, js_filtered_docs) -> Val; +field(#rctx{mango_eval_match=Val}, mango_eval_match) -> Val; +field(#rctx{get_kv_node=Val}, get_kv_node) -> Val; +field(#rctx{get_kp_node=Val}, get_kp_node) -> Val. + +curry_field(Field) -> + fun(Ele) -> field(Ele, Field) end. + +count_by(KeyFun) -> + group_by(KeyFun, fun(_) -> 1 end). + +group_by(KeyFun, ValFun) -> + group_by(KeyFun, ValFun, fun erlang:'+'/2). + +%% eg: group_by(mfa, docs_read). +%% eg: group_by(fun(#rctx{mfa=MFA,docs_read=DR}) -> {MFA, DR} end, ioq_calls). +%% eg: ^^ or: group_by([mfa, docs_read], ioq_calls). +%% eg: group_by([username, dbname, mfa], docs_read). +%% eg: group_by([username, dbname, mfa], ioq_calls). +%% eg: group_by([username, dbname, mfa], js_filters). +group_by(KeyL, ValFun, AggFun) when is_list(KeyL) -> + KeyFun = fun(Ele) -> list_to_tuple([field(Ele, Key) || Key <- KeyL]) end, + group_by(KeyFun, ValFun, AggFun); +group_by(Key, ValFun, AggFun) when is_atom(Key) -> + group_by(curry_field(Key), ValFun, AggFun); +group_by(KeyFun, Val, AggFun) when is_atom(Val) -> + group_by(KeyFun, curry_field(Val), AggFun); +group_by(KeyFun, ValFun, AggFun) -> + FoldFun = fun(Ele, Acc) -> + Key = KeyFun(Ele), + Val = ValFun(Ele), + CurrVal = maps:get(Key, Acc, 0), + NewVal = AggFun(CurrVal, Val), + %% TODO: should we skip here? how to make this optional? + case NewVal > 0 of + true -> + maps:put(Key, NewVal, Acc); + false -> + Acc + end + end, + ets:foldl(FoldFun, #{}, ?MODULE). + +%% Sorts largest first +sorted(Map) when is_map(Map) -> + lists:sort(fun({_K1, A}, {_K2, B}) -> B < A end, maps:to_list(Map)). + +shortened(L) -> + lists:sublist(L, 10). + +%% eg: sorted_by([username, dbname, mfa], ioq_calls) +%% eg: sorted_by([dbname, mfa], doc_reads) +sorted_by(KeyFun) -> shortened(sorted(count_by(KeyFun))). +sorted_by(KeyFun, ValFun) -> shortened(sorted(group_by(KeyFun, ValFun))). +sorted_by(KeyFun, ValFun, AggFun) -> shortened(sorted(group_by(KeyFun, ValFun, AggFun))). + +to_json_list(List) when is_list(List) -> + lists:map(fun csrt_util:to_json/1, List). + diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl new file mode 100644 index 00000000000..a3135b97ef8 --- /dev/null +++ b/src/couch_stats/src/csrt_server.erl @@ -0,0 +1,198 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_server). + +-behaviour(gen_server). + +-export([ + start_link/0, + init/1, + handle_call/3, + handle_cast/2 +]). + +-export([ + create_pid_ref/0, + create_resource/1, + destroy_resource/1, + get_resource/1, + get_context_type/1, + inc/2, + inc/3, + new_context/2, + set_context_dbname/2, + set_context_username/2, + set_context_type/2, + update_counter/3 +]). + +-include_lib("stdlib/include/ms_transform.hrl"). +-include_lib("couch_stats_resource_tracker.hrl"). + + +-record(st, {}). + +%% +%% Public API +%% + +-spec create_pid_ref() -> pid_ref(). +create_pid_ref() -> + {self(), make_ref()}. + +%% +%% +%% Context lifecycle API +%% + +-spec new_context(Type :: rctx_type(), Nonce :: nonce()) -> rctx(). +new_context(Type, Nonce) -> + #rctx{ + nonce = Nonce, + pid_ref = create_pid_ref(), + type = Type + }. + +-spec set_context_dbname(DbName, PidRef) -> boolean() when + DbName :: dbname(), PidRef :: maybe_pid_ref(). +set_context_dbname(_, undefined) -> + false; +set_context_dbname(DbName, PidRef) -> + update_element(PidRef, [{#rctx.dbname, DbName}]). + +%%set_context_handler_fun(_, undefined) -> +%% ok; +%%set_context_handler_fun(Fun, PidRef) when is_function(Fun) -> +%% FProps = erlang:fun_info(Fun), +%% Mod = proplists:get_value(module, FProps), +%% Func = proplists:get_value(name, FProps), +%% #rctx{type=#coordinator{}=Coordinator} = get_resource(PidRef), +%% Update = [{#rctx.type, Coordinator#coordinator{mod=Mod, func=Func}}], +%% update_element(PidRef, Update). + +-spec set_context_username(UserName, PidRef) -> boolean() when + UserName :: username(), PidRef :: maybe_pid_ref(). +set_context_username(_, undefined) -> + ok; +set_context_username(UserName, PidRef) -> + update_element(PidRef, [{#rctx.username, UserName}]). + +-spec get_context_type(Rctx :: rctx()) -> rctx_type(). +get_context_type(#rctx{type=Type}) -> + Type. + +-spec set_context_type(Type, PidRef) -> boolean() when + Type :: rctx_type(), PidRef :: maybe_pid_ref(). +set_context_type(Type, PidRef) -> + update_element(PidRef, [{#rctx.type, Type}]). + +-spec create_resource(Rctx :: rctx()) -> true. +create_resource(#rctx{} = Rctx) -> + catch ets:insert(?MODULE, Rctx). + +-spec destroy_resource(PidRef :: maybe_pid_ref()) -> boolean(). +destroy_resource(undefined) -> + false; +destroy_resource({_,_}=PidRef) -> + catch ets:delete(?MODULE, PidRef). + +-spec get_resource(PidRef :: maybe_pid_ref()) -> maybe_rctx(). +get_resource(undefined) -> + undefined; +get_resource(PidRef) -> + catch case ets:lookup(?MODULE, PidRef) of + [#rctx{}=Rctx] -> + Rctx; + [] -> + undefined + end. + +-spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). +is_rctx_field(Field) -> + maps:is_key(Field, ?KEYS_TO_FIELDS). + +-spec get_rctx_field(Field :: rctx_field()) -> non_neg_integer(). +get_rctx_field(Field) -> + maps:get(Field, ?KEYS_TO_FIELDS). + +-spec update_counter(PidRef, Field, Count) -> non_neg_integer() when + PidRef :: maybe_pid_ref(), + Field :: rctx_field(), + Count :: non_neg_integer(). +update_counter(undefined, _Field, _Count) -> + 0; +update_counter({_Pid,_Ref}=PidRef, Field, Count) when Count >= 0 -> + %% TODO: mem3 crashes without catch, why do we lose the stats table? + case is_rctx_field(Field) of + true -> + Update = {get_rctx_field(Field), Count}, + catch ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref=PidRef}); + false -> + 0 + end. + +-spec inc(PidRef :: maybe_pid_ref(), Field :: rctx_field()) -> non_neg_integer(). +inc(PidRef, Field) -> + inc(PidRef, Field, 1). + +-spec inc(PidRef, Field, N) -> non_neg_integer() when + PidRef :: maybe_pid_ref(), + Field :: rctx_field(), + N :: non_neg_integer(). +inc(undefined, _Field, _) -> + 0; +inc(_PidRef, _Field, 0) -> + 0; +inc({_Pid,_Ref}=PidRef, Field, N) when is_integer(N) andalso N >= 0 -> + case is_rctx_field(Field) of + true -> + update_counter(PidRef, Field, N); + false -> + 0 + end. + +%% +%% gen_server callbacks +%% + +start_link() -> + gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). + +init([]) -> + ets:new(?MODULE, [ + named_table, + public, + {decentralized_counters, true}, + {write_concurrency, true}, + {read_concurrency, true}, + {keypos, #rctx.pid_ref} + ]), + {ok, #st{}}. + +handle_call(_, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State, 0}. + +%% +%% private functions +%% + +-spec update_element(PidRef :: maybe_pid_ref(), Updates :: [tuple()]) -> boolean(). +update_element(undefined, _Update) -> + false; +update_element({_Pid,_Ref}=PidRef, Update) -> + %% TODO: should we take any action when the update fails? + catch ets:update_element(?MODULE, PidRef, Update). + diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl new file mode 100644 index 00000000000..c386cf08ee0 --- /dev/null +++ b/src/couch_stats/src/csrt_util.erl @@ -0,0 +1,470 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_util). + +-export([ + is_enabled/0, + is_enabled_init_p/0, + conf_get/1, + conf_get/2, + get_pid_ref/0, + get_pid_ref/1, + set_pid_ref/1, + should_track_init_p/2, + tnow/0, + tutc/0, + tutc/1 +]). + +%% JSON Conversion API +-export([ + convert_type/1, + convert_pidref/1, + convert_pid/1, + convert_ref/1, + to_json/1 +]). + +%% Delta API +-export([ + add_delta/2, + extract_delta/1, + get_delta/1, + get_delta_a/0, + get_delta_zero/0, + maybe_add_delta/1, + maybe_add_delta/2, + make_delta/1, + make_dt/2, + make_dt/3, + rctx_delta/2, + set_delta_a/1, + set_delta_zero/1 +]). + +%% Extra niceties and testing facilities +-export([ + set_fabric_init_p/2, + set_fabric_init_p/3, + map_to_rctx/1, + field/2 +]). + + +-include_lib("couch_stats_resource_tracker.hrl"). + +-spec is_enabled() -> boolean(). +is_enabled() -> + config:get_boolean(?CSRT, "enabled", true). + +-spec is_enabled_init_p() -> boolean(). +is_enabled_init_p() -> + config:get_boolean(?CSRT_INIT_P, "enabled", true). + +-spec should_track_init_p(Mod :: atom(), Func :: atom()) -> boolean(). +should_track_init_p(fabric_rpc, Func) -> + config:get_boolean(?CSRT_INIT_P, fabric_conf_key(Func), false); +should_track_init_p(_Mod, _Func) -> + false. + +-spec conf_get(Key :: list()) -> list(). +conf_get(Key) -> + conf_get(Key, undefined). + +-spec conf_get(Key :: list(), Default :: list()) -> list(). +conf_get(Key, Default) -> + config:get(?CSRT, Key, Default). + +%% Monotnonic time now in native format using time forward only event tracking +-spec tnow() -> integer(). +tnow() -> + erlang:monotonic_time(). + +%% Get current system time in UTC RFC 3339 format +-spec tutc() -> calendar:rfc3339_string(). +tutc() -> + tutc(tnow()). + +%% Convert a integer system time in milliseconds into UTC RFC 3339 format +-spec tutc(Time :: integer()) -> calendar:rfc3339_string(). +tutc(Time0) when is_integer(Time0) -> + Unit = millisecond, + Time1 = Time0 + erlang:time_offset(), + Time = erlang:convert_time_unit(Time1, native, Unit), + calendar:system_time_to_rfc3339(Time, [{unit, Unit}, {offset, "z"}]). + +%% Returns dt (delta time) in microseconds +%% @equiv make_dt(A, B, microsecond) +-spec make_dt(A, B) -> pos_integer() when + A :: integer(), + B :: integer(). +make_dt(A, B) -> + make_dt(A, B, microsecond). + +%% Returns monotonic dt (delta time) in specified time_unit() +-spec make_dt(A, B, Unit) -> pos_integer() when + A :: integer(), + B :: integer(), + Unit :: erlang:time_unit(). +make_dt(A, A, _Unit) when is_integer(A) -> + %% Handle edge case when monotonic_time()'s are equal + %% Always return a non zero value so we don't divide by zero + %% This always returns 1, independent of unit, as that's the smallest + %% possible positive integer value delta. + 1; +make_dt(A, B, Unit) when is_integer(A) andalso is_integer(B) andalso B > A -> + A1 = erlang:convert_time_unit(A, native, Unit), + B1 = erlang:convert_time_unit(B, native, Unit), + B1 - A1. + +%% +%% Conversion API for outputting JSON +%% + +-spec convert_type(T) -> binary() | null when + T :: #coordinator{} | #rpc_worker{} | undefined. +convert_type(#coordinator{method=Verb0, path=Path, mod=M0, func=F0}) -> + M = atom_to_binary(M0), + F = atom_to_binary(F0), + Verb = atom_to_binary(Verb0), + <<"coordinator-{", M/binary, ":", F/binary, "}:", Verb/binary, ":", Path/binary>>; +convert_type(#rpc_worker{mod=M0, func=F0, from=From0}) -> + M = atom_to_binary(M0), + F = atom_to_binary(F0), + From = convert_pidref(From0), + <<"rpc_worker-{", From/binary, "}:", M/binary, ":", F/binary>>; +convert_type(undefined) -> + null. + +-spec convert_pidref(PidRef) -> binary() | null when + PidRef :: {A :: pid(), B :: reference()} | undefined. +convert_pidref({Parent0, ParentRef0}) -> + Parent = convert_pid(Parent0), + ParentRef = convert_ref(ParentRef0), + <>; +%%convert_pidref(null) -> +%% null; +convert_pidref(undefined) -> + null. + +-spec convert_pid(Pid :: pid()) -> binary(). +convert_pid(Pid) when is_pid(Pid) -> + list_to_binary(pid_to_list(Pid)). + +-spec convert_ref(Ref :: reference()) -> binary(). +convert_ref(Ref) when is_reference(Ref) -> + list_to_binary(ref_to_list(Ref)). + +-spec to_json(Rctx :: rctx()) -> map(). +to_json(#rctx{}=Rctx) -> + #{ + updated_at => tutc(Rctx#rctx.updated_at), + started_at => tutc(Rctx#rctx.started_at), + pid_ref => convert_pidref(Rctx#rctx.pid_ref), + nonce => Rctx#rctx.nonce, + dbname => Rctx#rctx.dbname, + username => Rctx#rctx.username, + db_open => Rctx#rctx.db_open, + docs_read => Rctx#rctx.docs_read, + docs_written => Rctx#rctx.docs_written, + js_filter => Rctx#rctx.js_filter, + js_filtered_docs => Rctx#rctx.js_filtered_docs, + rows_read => Rctx#rctx.rows_read, + type => convert_type(Rctx#rctx.type), + get_kp_node => Rctx#rctx.get_kp_node, + get_kv_node => Rctx#rctx.get_kv_node, + write_kp_node => Rctx#rctx.write_kp_node, + write_kv_node => Rctx#rctx.write_kv_node, + changes_returned => Rctx#rctx.changes_returned, + changes_processed => Rctx#rctx.changes_processed, + ioq_calls => Rctx#rctx.ioq_calls + }. + +%% NOTE: this does not do the inverse of to_json, should it conver types? +-spec map_to_rctx(Map :: map()) -> rctx(). +map_to_rctx(Map) -> + maps:fold(fun map_to_rctx_field/3, #rctx{}, Map). + +-spec map_to_rctx_field(Field :: rctx_field(), Val :: any(), Rctx :: rctx()) -> rctx(). +map_to_rctx_field(updated_at, Val, Rctx) -> + Rctx#rctx{updated_at = Val}; +map_to_rctx_field(started_at, Val, Rctx) -> + Rctx#rctx{started_at = Val}; +map_to_rctx_field(pid_ref, Val, Rctx) -> + Rctx#rctx{pid_ref = Val}; +map_to_rctx_field(nonce, Val, Rctx) -> + Rctx#rctx{nonce = Val}; +map_to_rctx_field(dbname, Val, Rctx) -> + Rctx#rctx{dbname = Val}; +map_to_rctx_field(username, Val, Rctx) -> + Rctx#rctx{username = Val}; +map_to_rctx_field(db_open, Val, Rctx) -> + Rctx#rctx{db_open = Val}; +map_to_rctx_field(docs_read, Val, Rctx) -> + Rctx#rctx{docs_read = Val}; +map_to_rctx_field(docs_written, Val, Rctx) -> + Rctx#rctx{docs_written = Val}; +map_to_rctx_field(js_filter, Val, Rctx) -> + Rctx#rctx{js_filter = Val}; +map_to_rctx_field(js_filtered_docs, Val, Rctx) -> + Rctx#rctx{js_filtered_docs = Val}; +map_to_rctx_field(rows_read, Val, Rctx) -> + Rctx#rctx{rows_read = Val}; +map_to_rctx_field(type, Val, Rctx) -> + Rctx#rctx{type = Val}; +map_to_rctx_field(get_kp_node, Val, Rctx) -> + Rctx#rctx{get_kp_node = Val}; +map_to_rctx_field(get_kv_node, Val, Rctx) -> + Rctx#rctx{get_kv_node = Val}; +map_to_rctx_field(write_kp_node, Val, Rctx) -> + Rctx#rctx{write_kp_node = Val}; +map_to_rctx_field(write_kv_node, Val, Rctx) -> + Rctx#rctx{write_kv_node = Val}; +map_to_rctx_field(changes_returned, Val, Rctx) -> + Rctx#rctx{changes_returned = Val}; +map_to_rctx_field(changes_processed, Val, Rctx) -> + Rctx#rctx{changes_processed = Val}; +map_to_rctx_field(ioq_calls, Val, Rctx) -> + Rctx#rctx{ioq_calls = Val}. + +-spec field(Field :: rctx_field(), Rctx :: rctx()) -> any(). +field(updated_at, #rctx{updated_at = Val}) -> + Val; +field(started_at, #rctx{started_at = Val}) -> + Val; +field(pid_ref, #rctx{pid_ref = Val}) -> + Val; +field(nonce, #rctx{nonce = Val}) -> + Val; +field(dbname, #rctx{dbname = Val}) -> + Val; +field(username, #rctx{username = Val}) -> + Val; +field(db_open, #rctx{db_open = Val}) -> + Val; +field(docs_read, #rctx{docs_read = Val}) -> + Val; +field(docs_written, #rctx{docs_written = Val}) -> + Val; +field(js_filter, #rctx{js_filter = Val}) -> + Val; +field(js_filtered_docs, #rctx{js_filtered_docs = Val}) -> + Val; +field(rows_read, #rctx{rows_read = Val}) -> + Val; +field(type, #rctx{type = Val}) -> + Val; +field(get_kp_node, #rctx{get_kp_node = Val}) -> + Val; +field(get_kv_node, #rctx{get_kv_node = Val}) -> + Val; +field(changes_returned, #rctx{changes_returned = Val}) -> + Val; +field(changes_processed, #rctx{changes_processed = Val}) -> + Val; +field(ioq_calls, #rctx{ioq_calls = Val}) -> + Val. + +add_delta({A}, Delta) -> {A, Delta}; +add_delta({A, B}, Delta) -> {A, B, Delta}; +add_delta({A, B, C}, Delta) -> {A, B, C, Delta}; +add_delta({A, B, C, D}, Delta) -> {A, B, C, D, Delta}; +add_delta({A, B, C, D, E}, Delta) -> {A, B, C, D, E, Delta}; +add_delta({A, B, C, D, E, F}, Delta) -> {A, B, C, D, E, F, Delta}; +add_delta({A, B, C, D, E, F, G}, Delta) -> {A, B, C, D, E, F, G, Delta}; +add_delta(T, _Delta) -> T. + +extract_delta({A, {delta, Delta}}) -> {{A}, Delta}; +extract_delta({A, B, {delta, Delta}}) -> {{A, B}, Delta}; +extract_delta({A, B, C, {delta, Delta}}) -> {{A, B, C}, Delta}; +extract_delta({A, B, C, D, {delta, Delta}}) -> {{A, B, C, D}, Delta}; +extract_delta({A, B, C, D, E, {delta, Delta}}) -> {{A, B, C, D, E}, Delta}; +extract_delta({A, B, C, D, E, F, {delta, Delta}}) -> {{A, B, C, D, E, F}, Delta}; +extract_delta({A, B, C, D, E, F, G, {delta, Delta}}) -> {{A, B, C, D, E, F, G}, Delta}; +extract_delta(T) -> {T, undefined}. + +-spec get_delta(PidRef :: maybe_pid_ref()) -> tagged_delta(). +get_delta(PidRef) -> + {delta, make_delta(PidRef)}. + +maybe_add_delta(T) -> + case is_enabled() of + false -> + T; + true -> + maybe_add_delta_int(T, get_delta(get_pid_ref())) + end. + +%% Allow for externally provided Delta in error handling scenarios +%% eg in cases like rexi_server:notify_caller/3 +maybe_add_delta(T, Delta) -> + case is_enabled() of + false -> + T; + true -> + maybe_add_delta_int(T, Delta) + end. + +maybe_add_delta_int(T, undefined) -> + T; +maybe_add_delta_int(T, Delta) when is_map(Delta) -> + maybe_add_delta_int(T, {delta, Delta}); +maybe_add_delta_int(T, {delta, _} = Delta) -> + add_delta(T, Delta). + +-spec make_delta(PidRef :: maybe_pid_ref()) -> maybe_delta(). +make_delta(undefined) -> + undefined; +make_delta(PidRef) -> + TA = get_delta_a(), + TB = csrt_server:get_resource(PidRef), + Delta = rctx_delta(TA, TB), + set_delta_a(TB), + Delta. + +-spec rctx_delta(TA :: Rctx, TB :: Rctx) -> map(). +rctx_delta(#rctx{}=TA, #rctx{}=TB) -> + Delta = #{ + docs_read => TB#rctx.docs_read - TA#rctx.docs_read, + docs_written => TB#rctx.docs_written - TA#rctx.docs_written, + js_filter => TB#rctx.js_filter - TA#rctx.js_filter, + js_filtered_docs => TB#rctx.js_filtered_docs - TA#rctx.js_filtered_docs, + rows_read => TB#rctx.rows_read - TA#rctx.rows_read, + changes_returned => TB#rctx.changes_returned - TA#rctx.changes_returned, + changes_processed => TB#rctx.changes_processed - TA#rctx.changes_processed, + get_kp_node => TB#rctx.get_kp_node - TA#rctx.get_kp_node, + get_kv_node => TB#rctx.get_kv_node - TA#rctx.get_kv_node, + db_open => TB#rctx.db_open - TA#rctx.db_open, + ioq_calls => TB#rctx.ioq_calls - TA#rctx.ioq_calls, + dt => make_dt(TA#rctx.updated_at, TB#rctx.updated_at) + }, + %% TODO: reevaluate this decision + %% Only return non zero (and also positive) delta fields + %% NOTE: this can result in Delta's of the form #{dt => 1} + maps:filter(fun(_K,V) -> V > 0 end, Delta); +rctx_delta(_, _) -> + undefined. + +-spec get_delta_a() -> maybe_rctx(). +get_delta_a() -> + erlang:get(?DELTA_TA). + +-spec get_delta_zero() -> maybe_rctx(). +get_delta_zero() -> + erlang:get(?DELTA_TZ). + +-spec set_delta_a(TA :: rctx()) -> maybe_rctx(). +set_delta_a(TA) -> + erlang:put(?DELTA_TA, TA). + +-spec set_delta_zero(TZ :: rctx()) -> maybe_rctx(). +set_delta_zero(TZ) -> + erlang:put(?DELTA_TZ, TZ). + +-spec get_pid_ref() -> maybe_pid_ref(). +get_pid_ref() -> + get(?PID_REF). + +-spec get_pid_ref(Rctx :: rctx()) -> pid_ref(). +get_pid_ref(#rctx{pid_ref=PidRef}) -> + PidRef; +get_pid_ref(R) -> + throw({unexpected, R}). + +-spec set_pid_ref(PidRef :: pid_ref()) -> pid_ref(). +set_pid_ref(PidRef) -> + erlang:put(?PID_REF, PidRef), + PidRef. + +%% @equiv set_fabric_init_p(Func, Enabled, true). +-spec set_fabric_init_p(Func :: atom(), Enabled :: boolean()) -> ok. +set_fabric_init_p(Func, Enabled) -> + set_fabric_init_p(Func, Enabled, true). + +%% Expose Persist for use in test cases outside this module +-spec set_fabric_init_p(Func, Enabled, Persist) -> ok when + Func :: atom(), Enabled :: boolean(), Persist :: boolean(). +set_fabric_init_p(Func, Enabled, Persist) -> + Key = fabric_conf_key(Func), + ok = config:set_boolean(?CSRT_INIT_P, Key, Enabled, Persist). + +-spec fabric_conf_key(Key :: atom()) -> string(). +fabric_conf_key(Key) -> + %% Double underscore to separate Mod and Func + "fabric_rpc__" ++ atom_to_list(Key). + +-ifdef(TEST). + +-include_lib("couch/include/couch_eunit.hrl"). + +couch_stats_resource_tracker_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_should_track_init_p), + ?TDEF_FE(t_should_track_init_p_empty), + ?TDEF_FE(t_should_track_init_p_disabled), + ?TDEF_FE(t_should_not_track_init_p) + ] + }. + +setup() -> + test_util:start_couch(). + +teardown(Ctx) -> + test_util:stop_couch(Ctx). + +t_should_track_init_p(_) -> + enable_init_p(), + [?assert(should_track_init_p(M, F), {M, F}) || [M, F] <- base_metrics()]. + +t_should_track_init_p_empty(_) -> + config:set(?CSRT_INIT_P, "enabled", "true", false), + [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. + +t_should_track_init_p_disabled(_) -> + config:set(?CSRT_INIT_P, "enabled", "false", false), + [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. + +t_should_not_track_init_p(_) -> + enable_init_p(), + Metrics = [ + [couch_db, name], + [couch_db, get_after_doc_read_fun], + [couch_db, open], + [fabric_rpc, get_purge_seq] + ], + [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- Metrics]. + +enable_init_p() -> + enable_init_p(base_metrics()). + +enable_init_p(Metrics) -> + config:set(?CSRT_INIT_P, "enabled", "true", false), + [set_fabric_init_p(F, true, false) || [_, F] <- Metrics]. + +base_metrics() -> + [ + [fabric_rpc, all_docs], + [fabric_rpc, changes], + [fabric_rpc, map_view], + [fabric_rpc, reduce_view], + [fabric_rpc, get_all_security], + [fabric_rpc, open_doc], + [fabric_rpc, update_docs], + [fabric_rpc, open_shard] + ]. + +-endif. diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl new file mode 100644 index 00000000000..648b7601cfe --- /dev/null +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -0,0 +1,345 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_logger_tests). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch_mrview/include/couch_mrview.hrl"). + +-define(RCTX_RANGE, 1000). +-define(RCTX_COUNT, 10000). + +%% Dirty hack for hidden records as .hrl is only in src/ +-define(RCTX_RPC, {rpc_worker, foo, bar, {self(), make_ref()}}). +-define(RCTX_COORDINATOR, {coordinator, foo, bar, 'GET', "/foo/_all_docs"}). + +-define(THRESHOLD_DBNAME, <<"foo">>). +-define(THRESHOLD_DBNAME_IO, 91). +-define(THRESHOLD_DOCS_READ, 123). +-define(THRESHOLD_IOQ_CALLS, 439). +-define(THRESHOLD_ROWS_READ, 143). +-define(THRESHOLD_CHANGES, 79). + +csrt_logger_works_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_do_report), + ?TDEF_FE(t_do_lifetime_report), + ?TDEF_FE(t_do_status_report) + ] + }. + + +csrt_logger_matchers_test_() -> + { + foreach, + fun setup/0, + fun teardown/1, + [ + ?TDEF_FE(t_matcher_on_dbname), + ?TDEF_FE(t_matcher_on_dbnames_io), + ?TDEF_FE(t_matcher_on_docs_read), + ?TDEF_FE(t_matcher_on_docs_written), + ?TDEF_FE(t_matcher_on_rows_read), + ?TDEF_FE(t_matcher_on_worker_changes_processed), + ?TDEF_FE(t_matcher_on_ioq_calls), + ?TDEF_FE(t_matcher_on_nonce) + ] + }. + +make_docs(Count) -> + lists:map( + fun(I) -> + #doc{ + id = ?l2b("foo_" ++ integer_to_list(I)), + body={[{<<"value">>, I}]} + } + end, + lists:seq(1, Count)). + +setup() -> + Ctx = test_util:start_couch([fabric, couch_stats]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, 8}, {n, 1}]), + Docs = make_docs(100), + Opts = [], + {ok, _} = fabric:update_docs(DbName, Docs, Opts), + Method = 'GET', + Path = "/" ++ ?b2l(DbName) ++ "/_all_docs", + Nonce = couch_util:to_hex(crypto:strong_rand_bytes(5)), + Req = #httpd{method=Method, nonce=Nonce}, + {_, _} = PidRef = csrt:create_coordinator_context(Req, Path), + MArgs = #mrargs{include_docs = false}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = config:set("csrt_logger.matchers_threshold", "docs_read", integer_to_list(?THRESHOLD_DOCS_READ), false), + ok = config:set("csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false), + ok = config:set("csrt_logger.matchers_threshold", "rows_read", integer_to_list(?THRESHOLD_ROWS_READ), false), + ok = config:set("csrt_logger.matchers_threshold", "worker_changes_processed", integer_to_list(?THRESHOLD_CHANGES), false), + ok = config:set("csrt_logger.dbnames_io", "foo", integer_to_list(?THRESHOLD_DBNAME_IO), false), + ok = config:set("csrt_logger.dbnames_io", "bar", integer_to_list(?THRESHOLD_DBNAME_IO), false), + ok = config:set("csrt_logger.dbnames_io", "foo/bar", integer_to_list(?THRESHOLD_DBNAME_IO), false), + csrt_logger:reload_matchers(), + #{ctx => Ctx, dbname => DbName, rctx => Rctx, rctxs => rctxs()}. + +teardown(#{ctx := Ctx, dbname := DbName}) -> + ok = fabric:delete_db(DbName, [?ADMIN_CTX]), + test_util:stop_couch(Ctx). + +rctx_gen() -> + rctx_gen(#{}). + +rctx_gen(Opts0) -> + DbnameGen = one_of([<<"foo">>, <<"bar">>, ?tempdb]), + TypeGen = one_of([?RCTX_RPC, ?RCTX_COORDINATOR]), + R = fun() -> rand:uniform(?RCTX_RANGE) end, + R10 = fun() -> 3 + rand:uniform(round(?RCTX_RANGE / 10)) end, + Occasional = one_of([0, 0, 0, 0, 0, R]), + Nonce = one_of(["9c54fa9283", "foobar7799", lists:duplicate(10, fun nonce/0)]), + Base = #{ + dbname => DbnameGen, + db_open => R10, + docs_read => R, + docs_written => Occasional, + get_kp_node => R10, + get_kv_node => R, + nonce => Nonce, + pid_ref => {self(), make_ref()}, + ioq_calls => R, + rows_read => R, + type => TypeGen, + '_do_changes' => true %% Hack because we need to modify both fields + }, + Opts = maps:merge(Base, Opts0), + csrt_util:map_to_rctx(maps:fold( + fun + %% Hack for changes because we need to modify both changes_processed + %% and changes_returned but the latter must be <= the former + ('_do_changes', V, Acc) -> + case V of + true -> + Processed = R(), + Returned = (one_of([0, 0, 1, Processed, rand:uniform(Processed)]))(), + maps:put( + changes_processed, + Processed, + maps:put(changes_returned, Returned, Acc)); + _ -> + Acc + end; + (K, F, Acc) when is_function(F) -> + maps:put(K, F(), Acc); + (K, V, Acc) -> + maps:put(K, V, Acc) + end, #{}, Opts + )). + +rctxs() -> + [rctx_gen() || _ <- lists:seq(1, ?RCTX_COUNT)]. + +t_do_report(#{rctx := Rctx}) -> + JRctx = csrt_util:to_json(Rctx), + ReportName = "foo", + ok = meck:new(couch_log), + ok = meck:expect(couch_log, report, fun(_, _) -> true end), + ?assert(csrt_logger:do_report(ReportName, Rctx), "CSRT _logger:do_report " ++ ReportName), + ?assert(meck:validate(couch_log), "CSRT do_report"), + ?assert(meck:validate(couch_log), "CSRT validate couch_log"), + ?assert( + meck:called(couch_log, report, [ReportName, JRctx]), + "CSRT couch_log:report" + ), + ok = meck:unload(couch_log). + +t_do_lifetime_report(#{rctx := Rctx}) -> + JRctx = csrt_util:to_json(Rctx), + ReportName = "csrt-pid-usage-lifetime", + ok = meck:new(couch_log), + ok = meck:expect(couch_log, report, fun(_, _) -> true end), + ?assert( + csrt_logger:do_lifetime_report(Rctx), + "CSRT _logger:do_report " ++ ReportName + ), + ?assert(meck:validate(couch_log), "CSRT validate couch_log"), + ?assert( + meck:called(couch_log, report, [ReportName, JRctx]), + "CSRT couch_log:report" + ), + ok = meck:unload(couch_log). + +t_do_status_report(#{rctx := Rctx}) -> + JRctx = csrt_util:to_json(Rctx), + ReportName = "csrt-pid-usage-status", + ok = meck:new(couch_log), + ok = meck:expect(couch_log, report, fun(_, _) -> true end), + ?assert(csrt_logger:do_status_report(Rctx), "csrt_logger:do_ " ++ ReportName), + ?assert(meck:validate(couch_log), "CSRT validate couch_log"), + ?assert( + meck:called(couch_log, report, [ReportName, JRctx]), + "CSRT couch_log:report" + ), + ok = meck:unload(couch_log). + +t_matcher_on_dbname(#{rctx := _Rctx, rctxs := Rctxs0}) -> + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{dbname => <<"foo">>}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_on(dbname, <<"foo">>), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("dbname"), Rctxs)), + "Dbname matcher on <<\"foo\">>" + ). + +t_matcher_on_docs_read(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_DOCS_READ, + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{docs_read => Threshold + 10}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_gte(docs_read, Threshold), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("docs_read"), Rctxs)), + "Docs read matcher" + ). + +t_matcher_on_docs_written(#{rctxs := Rctxs0}) -> + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{docs_written => 73}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_gte(docs_written, 1), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("docs_written"), Rctxs)), + "Docs written matcher" + ). + +t_matcher_on_rows_read(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_ROWS_READ, + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{rows_read => Threshold + 10}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_gte(rows_read, Threshold), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("rows_read"), Rctxs)), + "Rows read matcher" + ). + +t_matcher_on_worker_changes_processed(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_CHANGES, + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{changes_processed => Threshold + 10}) | Rctxs0], + ChangesFilter = fun(R) -> + Ret = csrt_util:field(changes_returned, R), + Proc = csrt_util:field(changes_processed, R), + (Proc - Ret) >= Threshold + end, + ?assertEqual( + lists:sort(lists:filter(ChangesFilter, Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("worker_changes_processed"), Rctxs)), + "Changes processed matcher" + ). + +t_matcher_on_ioq_calls(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_IOQ_CALLS, + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{ioq_calls => Threshold + 10}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_gte(ioq_calls, Threshold), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("ioq_calls"), Rctxs)), + "IOQ calls matcher" + ). + +t_matcher_on_nonce(#{rctxs := Rctxs0}) -> + Nonce = "foobar7799", + %% Make sure we have at least one match + Rctxs = [rctx_gen(#{nonce => Nonce}) | Rctxs0], + %% Nonce requires dynamic matcher as it's a static match + %% TODO: add pattern based nonce matching + MSpec = csrt_logger:matcher_on_nonce(Nonce), + CompMSpec = ets:match_spec_compile(MSpec), + Matchers = #{"nonce" => {MSpec, CompMSpec}}, + IsMatch = fun(ARctx) -> csrt_logger:is_match(ARctx, Matchers) end, + ?assertEqual( + lists:sort(lists:filter(matcher_on(nonce, Nonce), Rctxs)), + lists:sort(lists:filter(IsMatch, Rctxs)), + "Rows read matcher" + ). + +t_matcher_on_dbnames_io(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_DBNAME_IO, + SThreshold = integer_to_list(Threshold), + DbFoo = "foo", + DbBar = "bar", + MatcherFoo = matcher_for_csrt("dbname_io__" ++ DbFoo ++ "__" ++ SThreshold), + MatcherBar = matcher_for_csrt("dbname_io__" ++ DbBar ++ "__" ++ SThreshold), + MatcherFooBar = matcher_for_csrt("dbname_io__foo/bar__" ++ SThreshold), + %% Add an extra Rctx with dbname foo/bar to ensure correct naming matches + ExtraRctx = rctx_gen(#{dbname => <<"foo/bar">>, get_kp_node => Threshold + 10}), + %% Make sure we have at least one match + Rctxs = [ExtraRctx, rctx_gen(#{ioq_calls => Threshold + 10}) | Rctxs0], + ?assertEqual( + lists:sort(lists:filter(matcher_for_dbname_io(DbFoo, Threshold), Rctxs)), + lists:sort(lists:filter(MatcherFoo, Rctxs)), + "dbname_io foo matcher" + ), + ?assertEqual( + lists:sort(lists:filter(matcher_for_dbname_io(DbBar, Threshold), Rctxs)), + lists:sort(lists:filter(MatcherBar, Rctxs)), + "dbname_io bar matcher" + ), + ?assertEqual( + [ExtraRctx], + lists:sort(lists:filter(MatcherFooBar, Rctxs)), + "dbname_io foo/bar matcher" + ). + +load_rctx(PidRef) -> + timer:sleep(50), %% Add slight delay to accumulate RPC response deltas + csrt:get_resource(PidRef). + +view_cb({row, Row}, Acc) -> + {ok, [Row | Acc]}; +view_cb(_Msg, Acc) -> + {ok, Acc}. + +matcher_gte(Field, Value) -> + matcher_for(Field, Value, fun erlang:'>='/2). + +matcher_on(Field, Value) -> + matcher_for(Field, Value, fun erlang:'=:='/2). + +matcher_for(Field, Value, Op) -> + fun(Rctx) -> Op(csrt_util:field(Field, Rctx), Value) end. + +matcher_for_csrt(MatcherName) -> + Matchers = #{MatcherName => {_, _} = csrt_logger:get_matcher(MatcherName)}, + fun(Rctx) -> csrt_logger:is_match(Rctx, Matchers) end. + +matcher_for_dbname_io(Dbname0, Threshold) -> + Dbname = list_to_binary(Dbname0), + fun(Rctx) -> + DbnameA = csrt_util:field(dbname, Rctx), + Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read, changes_processed], + Vals = [{F, csrt_util:field(F, Rctx)} || F <- Fields], + Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun(V) -> V >= Threshold end, Vals) + end. + +nonce() -> + couch_util:to_hex(crypto:strong_rand_bytes(5)). + +one_of(L) -> + fun() -> + case lists:nth(rand:uniform(length(L)), L) of + F when is_function(F) -> + F(); + N -> + N + end + end. diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl new file mode 100644 index 00000000000..f3cf07a836a --- /dev/null +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -0,0 +1,575 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(csrt_server_tests). + +-include_lib("couch/include/couch_db.hrl"). +-include_lib("couch/include/couch_eunit.hrl"). +-include_lib("couch_mrview/include/couch_mrview.hrl"). + +-define(DOCS_COUNT, 100). +-define(DDOCS_COUNT, 1). +-define(DB_Q, 8). + +-define(DEBUG_ENABLED, false). + + +csrt_context_test_() -> + { + setup, + fun setup/0, + fun teardown/1, + with([ + ?TDEF(t_context_setting) + ]) + }. + +test_funs() -> + [ + ?TDEF_FE(t_all_docs_include_false), + ?TDEF_FE(t_all_docs_include_true), + ?TDEF_FE(t_all_docs_limit_zero), + ?TDEF_FE(t_get_doc), + ?TDEF_FE(t_put_doc), + ?TDEF_FE(t_delete_doc), + ?TDEF_FE(t_update_docs), + ?TDEF_FE(t_changes), + ?TDEF_FE(t_changes_limit_zero), + ?TDEF_FE(t_changes_filtered), + ?TDEF_FE(t_view_query), + ?TDEF_FE(t_view_query_include_docs) + ]. + +ddoc_test_funs() -> + [ + ?TDEF_FE(t_changes_js_filtered) + | test_funs() + ]. + +csrt_fabric_no_ddoc_test_() -> + { + "CSRT fabric tests with no DDoc present", + foreach, + fun setup/0, + fun teardown/1, + test_funs() + }. + +csrt_fabric_test_() -> + { + "CSRT fabric tests with a DDoc present", + foreach, + fun() -> setup_ddoc(<<"_design/foo">>, <<"bar">>) end, + fun teardown/1, + ddoc_test_funs() + }. + +make_docs(Count) -> + lists:map( + fun(I) -> + #doc{ + id = ?l2b("foo_" ++ integer_to_list(I)), + body={[{<<"value">>, I}]} + } + end, + lists:seq(1, Count)). + +setup() -> + Ctx = test_util:start_couch([fabric, couch_stats]), + DbName = ?tempdb(), + ok = fabric:create_db(DbName, [{q, ?DB_Q}, {n, 1}]), + Docs = make_docs(?DOCS_COUNT), + Opts = [], + {ok, _} = fabric:update_docs(DbName, Docs, Opts), + {Ctx, DbName, undefined}. + +teardown({Ctx, DbName, _View}) -> + ok = fabric:delete_db(DbName, [?ADMIN_CTX]), + test_util:stop_couch(Ctx). + +setup_ddoc(DDocId, ViewName) -> + {Ctx, DbName, undefined} = setup(), + DDoc = couch_doc:from_json_obj( + {[ + {<<"_id">>, DDocId}, + {<<"language">>, <<"javascript">>}, + { + <<"views">>, + {[{ + ViewName, + {[ + {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} + ]} + }]} + }, + { + <<"filters">>, + {[{ + <<"even">>, + <<"function(doc) { return (doc.value % 2 == 0); }">> + }]} + } + ]} + ), + {ok, _Rev} = fabric:update_doc(DbName, DDoc, [?ADMIN_CTX]), + {Ctx, DbName, {DDocId, ViewName}}. + +t_context_setting({_Ctx, _DbName, _View}) -> + false. + +t_all_docs_limit_zero({_Ctx, DbName, _View}) -> + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_all_docs" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + MArgs = #mrargs{include_docs = false, limit = 0}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => 0, + docs_read => 0, + docs_written => 0, + ioq_calls => assert_gt(), + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +t_all_docs_include_false({_Ctx, DbName, View}) -> + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_all_docs" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + MArgs = #mrargs{include_docs = false}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => docs_count(View), + docs_read => 0, + docs_written => 0, + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +t_all_docs_include_true({_Ctx, DbName, View}) -> + pdebug(dbname, DbName), + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_all_docs" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + MArgs = #mrargs{include_docs = true}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => docs_count(View), + docs_read => docs_count(View), + docs_written => 0, + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +t_update_docs({_Ctx, DbName, View}) -> + pdebug(dbname, DbName), + Context = #{ + method => 'POST', + path => "/" ++ ?b2l(DbName) + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + Docs = [#doc{id = ?l2b("bar_" ++ integer_to_list(I))} || I <- lists:seq(1, ?DOCS_COUNT)], + _Res = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]), + Rctx = load_rctx(PidRef), + pdebug(rctx, Rctx), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => 0, + docs_read => 0, + docs_written => ?DOCS_COUNT, + pid_ref => PidRef + }), + ok = ddoc_dependent_local_io_assert(Rctx, View), + ok = assert_teardown(PidRef). + +t_get_doc({_Ctx, DbName, _View}) -> + pdebug(dbname, DbName), + DocId = "foo_17", + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/" ++ DocId + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + _Res = fabric:open_doc(DbName, DocId, [?ADMIN_CTX]), + Rctx = load_rctx(PidRef), + pdebug(rctx, Rctx), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => 1, + rows_read => 0, + docs_read => 1, + docs_written => 0, + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx, io_sum), + ok = assert_teardown(PidRef). + + +t_put_doc({_Ctx, DbName, View}) -> + pdebug(dbname, DbName), + DocId = "bar_put_1919", + Context = #{ + method => 'PUT', + path => "/" ++ ?b2l(DbName) ++ "/" ++ DocId + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + Doc = #doc{id = ?l2b(DocId)}, + _Res = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]), + Rctx = load_rctx(PidRef), + pdebug(rctx, Rctx), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => 1, + rows_read => 0, + docs_read => 0, + docs_written => 1, + pid_ref => PidRef + }), + ok = ddoc_dependent_local_io_assert(Rctx, View), + ok = assert_teardown(PidRef). + +t_delete_doc({_Ctx, DbName, View}) -> + pdebug(dbname, DbName), + DocId = "foo_17", + {ok, Doc0} = fabric:open_doc(DbName, DocId, [?ADMIN_CTX]), + Doc = Doc0#doc{body = {[{<<"_deleted">>, true}]}}, + Context = #{ + method => 'DELETE', + path => "/" ++ ?b2l(DbName) ++ "/" ++ DocId + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + _Res = fabric:update_doc(DbName, Doc, [?ADMIN_CTX]), + Rctx = load_rctx(PidRef), + pdebug(rctx, Rctx), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => 1, + rows_read => 0, + docs_read => 0, + docs_written => 1, + pid_ref => PidRef + }), + ok = ddoc_dependent_local_io_assert(Rctx, View), + ok = assert_teardown(PidRef). + +t_changes({_Ctx, DbName, View}) -> + pdebug(dbname, DbName), + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_changes" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + _Res = fabric:changes(DbName, fun changes_cb/2, [], #changes_args{}), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => assert_gte(?DB_Q), + changes_returned => docs_count(View), + docs_read => 0, + docs_written => 0, + pid_ref => PidRef + }), + %% at least one rows_read and changes_returned per shard that has at least + %% one document in it + ?assert(maps:get(rows_read, Rctx) >= ?DB_Q, rows_read), + ?assert(maps:get(changes_returned, Rctx) >= ?DB_Q, changes_returned), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + + +t_changes_limit_zero({_Ctx, DbName, _View}) -> + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_changes" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + _Res = fabric:changes(DbName, fun changes_cb/2, [], #changes_args{limit=0}), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => false, + changes_returned => false, + docs_read => 0, + docs_written => 0, + pid_ref => PidRef + }), + ?assert(maps:get(rows_read, Rctx) >= ?DB_Q, rows), + ?assert(maps:get(changes_returned, Rctx) >= ?DB_Q, rows), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +%% TODO: stub in non JS filter with selector +t_changes_filtered({_Ctx, _DbName, _View}) -> + false. + +t_changes_js_filtered({_Ctx, DbName, {DDocId, _ViewName}=View}) -> + pdebug(dbname, DbName), + Method = 'GET', + Path = "/" ++ ?b2l(DbName) ++ "/_changes", + Context = #{ + method => Method, + path => Path + }, + {PidRef, Nonce} = coordinator_context(Context), + Req = {json_req, null}, + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + Filter = configure_filter(DbName, DDocId, Req), + Args = #changes_args{filter_fun = Filter}, + _Res = fabric:changes(DbName, fun changes_cb/2, [], Args), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => assert_gte(?DB_Q), + rows_read => assert_gte(docs_count(View)), + changes_returned => round(?DOCS_COUNT / 2), + docs_read => assert_gte(docs_count(View)), + docs_written => 0, + pid_ref => PidRef, + js_filter => docs_count(View), + js_filtered_docs => docs_count(View) + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +t_view_query({_Ctx, DbName, View}) -> + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_design/foo/_view/bar" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + MArgs = #mrargs{include_docs = false}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => docs_count(View), + docs_read => 0, + docs_written => 0, + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +t_view_query_include_docs({_Ctx, DbName, View}) -> + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/_design/foo/_view/bar" + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + MArgs = #mrargs{include_docs = true}, + _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), + Rctx = load_rctx(PidRef), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => ?DB_Q, + rows_read => docs_count(View), + docs_read => docs_count(View), + docs_written => 0, + pid_ref => PidRef + }), + ok = nonzero_local_io_assert(Rctx), + ok = assert_teardown(PidRef). + +assert_teardown(PidRef) -> + ?assertEqual(ok, csrt:destroy_context(PidRef)), + ?assertEqual(undefined, csrt:get_resource()), + %% Normally the tracker is responsible for destroying the resource + ?assertEqual(true, csrt_server:destroy_resource(PidRef)), + ?assertEqual(undefined, csrt:get_resource(PidRef)), + ok. + +view_cb({row, Row}, Acc) -> + {ok, [Row | Acc]}; +view_cb(_Msg, Acc) -> + {ok, Acc}. + +changes_cb({change, {Change}}, Acc) -> + {ok, [Change | Acc]}; +changes_cb(_Msg, Acc) -> + {ok, Acc}. + +pdebug(dbname, DbName) -> + case ?DEBUG_ENABLED =:= true of + true -> + ?debugFmt("DBNAME[~p]: ~p", [DbName, fabric:get_db_info(DbName)]); + false -> + ok + end; +pdebug(rctx, Rctx) -> + ?DEBUG_ENABLED andalso ?debugFmt("GOT RCTX: ~p~n", [Rctx]). + +pdbg(Str, Args) -> + ?DEBUG_ENABLED andalso ?debugFmt(Str, Args). + +convert_pidref({_, _}=PidRef) -> + csrt_util:convert_pidref(PidRef); +convert_pidref(PidRef) when is_binary(PidRef) -> + PidRef; +convert_pidref(false) -> + false. + +rctx_assert(Rctx, Asserts0) -> + DefaultAsserts = #{ + changes_returned => 0, + js_filter => 0, + js_filtered_docs => 0, + write_kp_node => 0, + write_kv_node => 0, + nonce => undefined, + db_open => 0, + rows_read => 0, + docs_read => 0, + docs_written => 0, + pid_ref => undefined + }, + Asserts = maps:merge( + DefaultAsserts, + maps:update_with(pid_ref, fun convert_pidref/1, Asserts0) + ), + ok = maps:foreach( + fun + (_K, false) -> + ok; + (K, Fun) when is_function(Fun) -> + Fun(K, maps:get(K, Rctx)); + (K, V) -> + case maps:get(K, Rctx) of + false -> + ok; + RV -> + pdbg("?assertEqual(~p, ~p, ~p)", [V, RV, K]), + ?assertEqual(V, RV, K) + end + end, + Asserts + ), + ok. + +%% Doc updates and others don't perform local IO, they funnel to another pid +zero_local_io_assert(Rctx) -> + ?assertEqual(0, maps:get(ioq_calls, Rctx)), + ?assertEqual(0, maps:get(get_kp_node, Rctx)), + ?assertEqual(0, maps:get(get_kv_node, Rctx)), + ok. + +nonzero_local_io_assert(Rctx) -> + nonzero_local_io_assert(Rctx, io_separate). + +%% io_sum for when get_kp_node=0 +nonzero_local_io_assert(Rctx, io_sum) -> + ?assert(maps:get(ioq_calls, Rctx) > 0), + #{ + get_kp_node := KPNodes, + get_kv_node := KVNodes + } = Rctx, + ?assert((KPNodes + KVNodes) > 0), + ok; +nonzero_local_io_assert(Rctx, io_separate) -> + ?assert(maps:get(ioq_calls, Rctx) > 0), + ?assert(maps:get(get_kp_node, Rctx) > 0), + ?assert(maps:get(get_kv_node, Rctx) > 0), + ok. + +ddoc_dependent_local_io_assert(Rctx, undefined) -> + zero_local_io_assert(Rctx); +ddoc_dependent_local_io_assert(Rctx, {_DDoc, _ViewName}) -> + nonzero_local_io_assert(Rctx, io_sum). + +coordinator_context(#{method := Method, path := Path}) -> + Nonce = couch_util:to_hex(crypto:strong_rand_bytes(5)), + Req = #httpd{method=Method, nonce=Nonce}, + {_, _} = PidRef = csrt:create_coordinator_context(Req, Path), + {PidRef, Nonce}. + +fresh_rctx_assert(Rctx, PidRef, Nonce) -> + pdebug(rctx, Rctx), + FreshAsserts = #{ + nonce => Nonce, + db_open => 0, + rows_read => 0, + docs_read => 0, + docs_written => 0, + pid_ref => PidRef + }, + rctx_assert(Rctx, FreshAsserts). + +assert_gt() -> + assert_gt(0). + +assert_gt(N) -> + fun(K, RV) -> ?assert(RV > N, {K, RV, N}) end. + +assert_gte(N) -> + fun(K, RV) -> ?assert(RV >= N, {K, RV, N}) end. + +docs_count(undefined) -> + ?DOCS_COUNT; +docs_count({_, _}) -> + ?DOCS_COUNT + ?DDOCS_COUNT. + +configure_filter(DbName, DDocId, Req) -> + configure_filter(DbName, DDocId, Req, <<"even">>). + +configure_filter(DbName, DDocId, Req, FName) -> + {ok, DDoc} = ddoc_cache:open_doc(DbName, DDocId), + DIR = fabric_util:doc_id_and_rev(DDoc), + Style = main_only, + {fetch, custom, Style, Req, DIR, FName}. + +load_rctx(PidRef) -> + timer:sleep(50), %% Add slight delay to accumulate RPC response deltas + csrt_util:to_json(csrt:get_resource(PidRef)). diff --git a/src/fabric/priv/stats_descriptions.cfg b/src/fabric/priv/stats_descriptions.cfg index d12aa0c8480..9ab054bf038 100644 --- a/src/fabric/priv/stats_descriptions.cfg +++ b/src/fabric/priv/stats_descriptions.cfg @@ -26,3 +26,53 @@ {type, counter}, {desc, <<"number of write quorum errors">>} ]}. + + +%% fabric_rpc worker stats +%% TODO: decide on which naming scheme: +%% {[fabric_rpc, get_all_security, spawned], [ +%% {[fabric_rpc, spawned, get_all_security], [ +{[fabric_rpc, get_all_security, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker get_all_security spawns">>} +]}. +{[fabric_rpc, open_doc, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker open_doc spawns">>} +]}. +{[fabric_rpc, all_docs, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker all_docs spawns">>} +]}. +{[fabric_rpc, update_docs, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker update_docs spawns">>} +]}. +{[fabric_rpc, map_view, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker map_view spawns">>} +]}. +{[fabric_rpc, reduce_view, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker reduce_view spawns">>} +]}. +{[fabric_rpc, open_shard, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker open_shard spawns">>} +]}. +{[fabric_rpc, changes, spawned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker changes spawns">>} +]}. +{[fabric_rpc, changes, processed], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker changes row invocations">>} +]}. +{[fabric_rpc, changes, returned], [ + {type, counter}, + {desc, <<"number of fabric_rpc worker changes rows returned">>} +]}. +{[fabric_rpc, view, rows_read], [ + {type, counter}, + {desc, <<"number of fabric_rpc view_cb row invocations">>} +]}. diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index 67f529e0935..afa1656009f 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -284,6 +284,7 @@ get_missing_revs(DbName, IdRevsList, Options) -> with_db(DbName, Options, {couch_db, get_missing_revs, [IdRevsList]}). update_docs(DbName, Docs0, Options) -> + csrt:docs_written(length(Docs0)), {Docs1, Type} = case couch_util:get_value(read_repair, Options) of NodeRevs when is_list(NodeRevs) -> @@ -493,6 +494,11 @@ view_cb({meta, Meta}, Acc) -> ok = rexi:stream2({meta, Meta}), {ok, Acc}; view_cb({row, Props}, #mrargs{extra = Options} = Acc) -> + %% TODO: distinguish between rows and docs + %% TODO: wire in csrt tracking + %% TODO: distinguish between all_docs vs view call + couch_stats:increment_counter([fabric_rpc, view, rows_read]), + %%csrt:inc(rows_read), % Adding another row ViewRow = fabric_view_row:from_props(Props, Options), ok = rexi:stream2(ViewRow), @@ -529,6 +535,7 @@ changes_enumerator(#full_doc_info{} = FDI, Acc) -> changes_enumerator(#doc_info{id = <<"_local/", _/binary>>, high_seq = Seq}, Acc) -> {ok, Acc#fabric_changes_acc{seq = Seq, pending = Acc#fabric_changes_acc.pending - 1}}; changes_enumerator(DocInfo, Acc) -> + couch_stats:increment_counter([fabric_rpc, changes, processed]), #fabric_changes_acc{ db = Db, args = #changes_args{ @@ -569,6 +576,7 @@ changes_enumerator(DocInfo, Acc) -> {ok, Acc#fabric_changes_acc{seq = Seq, pending = Pending - 1}}. changes_row(Changes, Docs, DocInfo, Acc) -> + couch_stats:increment_counter([fabric_rpc, changes, returned]), #fabric_changes_acc{db = Db, pending = Pending, epochs = Epochs} = Acc, #doc_info{id = Id, high_seq = Seq, revs = [#rev_info{deleted = Del} | _]} = DocInfo, {change, [ @@ -667,6 +675,14 @@ clean_stack(S) -> ). set_io_priority(DbName, Options) -> + csrt:set_context_dbname(DbName), + %% TODO: better approach here than using proplists? + case proplists:get_value(user_ctx, Options) of + undefined -> + ok; + #user_ctx{name = UserName} -> + csrt:set_context_username(UserName) + end, case lists:keyfind(io_priority, 1, Options) of {io_priority, Pri} -> erlang:put(io_priority, Pri); diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index 4acb65c739a..f93644d2e93 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -136,24 +136,36 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]}, Ref = rexi:cast(Node, self(), MFA, [sync]), try - receive - {Ref, {ok, Db}} -> - {ok, Db}; - {Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} -> - throw(Error); - {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> - throw(Error); - {Ref, Reason} -> - couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), - get_shard(Rest, Opts, Timeout, Factor) - after Timeout -> - couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), - get_shard(Rest, Opts, Factor * Timeout, Factor) - end + await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) after rexi_monitor:stop(Mon) end. +await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) -> + receive + Msg0 -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), + case Msg of + {Ref, {ok, Db}} -> + {ok, Db}; + {Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} -> + throw(Error); + {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> + throw(Error); + {Ref, Reason} -> + couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), + get_shard(Rest, Opts, Timeout, Factor); + %% {OldRef, {ok, Db}} -> <-- stale db resp that got here late, should we do something? + _ -> + %% Got a message from an old Ref that timed out, try again + await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) + end + after Timeout -> + couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), + get_shard(Rest, Opts, Factor * Timeout, Factor) + end. + get_db_timeout(N, Factor, MinTimeout, infinity) -> % MaxTimeout may be infinity so we just use the largest Erlang small int to % avoid blowing up the arithmetic diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl index 07e6b1d4220..c7a36fbe342 100644 --- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl @@ -263,6 +263,8 @@ rpc_update_doc(DbName, Doc, Opts) -> Reply = test_util:wait(fun() -> receive {Ref, Reply} -> + Reply; + {Ref, Reply, {delta, _}} -> Reply after 0 -> wait diff --git a/src/fabric/test/eunit/fabric_rpc_tests.erl b/src/fabric/test/eunit/fabric_rpc_tests.erl index 16bb66badac..f5e4e52f691 100644 --- a/src/fabric/test/eunit/fabric_rpc_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_tests.erl @@ -101,7 +101,16 @@ t_no_config_db_create_fails_for_shard_rpc(DbName) -> receive Resp0 -> Resp0 end, - ?assertMatch({Ref, {'rexi_EXIT', {{error, missing_target}, _}}}, Resp). + case csrt:is_enabled() of + true -> + ?assertMatch( %% allow for {Ref, {rexi_EXIT, error}, {delta, D}} + {Ref, {'rexi_EXIT', {{error, missing_target}, _}}, _}, + Resp); + false -> + ?assertMatch( + {Ref, {'rexi_EXIT', {{error, missing_target}, _}}}, + Resp) + end. t_db_create_with_config(DbName) -> MDbName = mem3:dbname(DbName), diff --git a/src/ioq/src/ioq.erl b/src/ioq/src/ioq.erl index 8e38c2a0015..e8862857f7b 100644 --- a/src/ioq/src/ioq.erl +++ b/src/ioq/src/ioq.erl @@ -60,6 +60,7 @@ call_search(Fd, Msg, Metadata) -> call(Fd, Msg, Metadata). call(Fd, Msg, Metadata) -> + csrt:ioq_called(), case bypass(Msg, Metadata) of true -> gen_server:call(Fd, Msg, infinity); diff --git a/src/mango/src/mango_cursor_view.erl b/src/mango/src/mango_cursor_view.erl index 0928ae19311..e11c6941632 100644 --- a/src/mango/src/mango_cursor_view.erl +++ b/src/mango/src/mango_cursor_view.erl @@ -245,9 +245,11 @@ execute(#cursor{db = Db, index = Idx, execution_stats = Stats} = Cursor0, UserFu Result = case mango_idx:def(Idx) of all_docs -> + couch_stats:increment_counter([mango_cursor, view, all_docs]), CB = fun ?MODULE:handle_all_docs_message/2, fabric:all_docs(Db, DbOpts, CB, Cursor, Args); _ -> + couch_stats:increment_counter([mango_cursor, view, idx]), CB = fun ?MODULE:handle_message/2, % Normal view DDoc = ddocid(Idx), diff --git a/src/mango/src/mango_selector.erl b/src/mango/src/mango_selector.erl index 42031b7569d..d8d2c913c98 100644 --- a/src/mango/src/mango_selector.erl +++ b/src/mango/src/mango_selector.erl @@ -50,6 +50,7 @@ normalize(Selector) -> % This assumes that the Selector has been normalized. % Returns true or false. match(Selector, D) -> + %% TODO: wire in csrt tracking couch_stats:increment_counter([mango, evaluate_selector]), match_int(Selector, D). diff --git a/src/mem3/src/mem3_rpc.erl b/src/mem3/src/mem3_rpc.erl index 70fc797dad6..0711cfb67dc 100644 --- a/src/mem3/src/mem3_rpc.erl +++ b/src/mem3/src/mem3_rpc.erl @@ -378,20 +378,34 @@ rexi_call(Node, MFA, Timeout) -> Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]), Ref = rexi:cast(Node, self(), MFA, [sync]), try - receive - {Ref, {ok, Reply}} -> - Reply; - {Ref, Error} -> - erlang:error(Error); - {rexi_DOWN, Mon, _, Reason} -> - erlang:error({rexi_DOWN, {Node, Reason}}) - after Timeout -> - erlang:error(timeout) - end + wait_message(Node, Ref, Mon, Timeout) after rexi_monitor:stop(Mon) end. +wait_message(Node, Ref, Mon, Timeout) -> + receive + Msg -> + process_raw_message(Msg, Node, Ref, Mon, Timeout) + after Timeout -> + erlang:error(timeout) + end. + +process_raw_message(Msg0, Node, Ref, Mon, Timeout) -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), + case Msg of + {Ref, {ok, Reply}} -> + Reply; + {Ref, Error} -> + erlang:error(Error); + {rexi_DOWN, Mon, _, Reason} -> + erlang:error({rexi_DOWN, {Node, Reason}}); + Other -> + ?LOG_UNEXPECTED_MSG(Other), + wait_message(Node, Ref, Mon, Timeout) + end. + get_or_create_db(DbName, Options) -> mem3_util:get_or_create_db_int(DbName, Options). diff --git a/src/rexi/include/rexi.hrl b/src/rexi/include/rexi.hrl index a2d86b2ab54..a962f306917 100644 --- a/src/rexi/include/rexi.hrl +++ b/src/rexi/include/rexi.hrl @@ -11,6 +11,7 @@ % the License. -record(error, { + delta, timestamp, reason, mfa, diff --git a/src/rexi/src/rexi.erl b/src/rexi/src/rexi.erl index 02d3a9e5559..bb7c570375b 100644 --- a/src/rexi/src/rexi.erl +++ b/src/rexi/src/rexi.erl @@ -104,7 +104,7 @@ kill_all(NodeRefs) when is_list(NodeRefs) -> -spec reply(any()) -> any(). reply(Reply) -> {Caller, Ref} = get(rexi_from), - erlang:send(Caller, {Ref, Reply}). + erlang:send(Caller, csrt:maybe_add_delta({Ref, Reply})). %% Private function used by stream2 to initialize the stream. Message is of the %% form {OriginalRef, {self(),reference()}, Reply}, which enables the @@ -188,7 +188,7 @@ stream2(Msg, Limit, Timeout) -> {ok, Count} -> put(rexi_unacked, Count + 1), {Caller, Ref} = get(rexi_from), - erlang:send(Caller, {Ref, self(), Msg}), + erlang:send(Caller, csrt:maybe_add_delta({Ref, self(), Msg})), ok catch throw:timeout -> @@ -222,7 +222,11 @@ stream_ack(Client) -> %% ping() -> {Caller, _} = get(rexi_from), - erlang:send(Caller, {rexi, '$rexi_ping'}). + %% It is essential ping/0 includes deltas as otherwise long running + %% filtered queries will be silent on usage until they finally return + %% a row or no results. This delay is proportional to the database size, + %% so instead we make sure ping/0 keeps live stats flowing. + erlang:send(Caller, csrt:maybe_add_delta({rexi, '$rexi_ping'})). aggregate_server_queue_len() -> rexi_server_mon:aggregate_queue_len(rexi_server). diff --git a/src/rexi/src/rexi_monitor.erl b/src/rexi/src/rexi_monitor.erl index 7fe66db71d4..72f0985df80 100644 --- a/src/rexi/src/rexi_monitor.erl +++ b/src/rexi/src/rexi_monitor.erl @@ -35,6 +35,7 @@ start(Procs) -> %% messages from our mailbox. -spec stop(pid()) -> ok. stop(MonitoringPid) -> + unlink(MonitoringPid), MonitoringPid ! {self(), shutdown}, flush_down_messages(). diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index b2df65c7193..8ba1ee2e58c 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -102,12 +102,12 @@ handle_info({'DOWN', Ref, process, Pid, Error}, #st{workers = Workers} = St) -> case find_worker(Ref, Workers) of #job{worker_pid = Pid, worker = Ref, client_pid = CPid, client = CRef} = Job -> case Error of - #error{reason = {_Class, Reason}, stack = Stack} -> - notify_caller({CPid, CRef}, {Reason, Stack}), + #error{reason = {_Class, Reason}, stack = Stack, delta = Delta} -> + notify_caller({CPid, CRef}, {Reason, Stack}, Delta), St1 = save_error(Error, St), {noreply, remove_job(Job, St1)}; _ -> - notify_caller({CPid, CRef}, Error), + notify_caller({CPid, CRef}, Error, undefined), {noreply, remove_job(Job, St)} end; false -> @@ -134,15 +134,20 @@ init_p(From, MFA) -> string() | undefined ) -> any(). init_p(From, {M, F, A}, Nonce) -> + MFA = {M, F, length(A)}, put(rexi_from, From), - put('$initial_call', {M, F, length(A)}), + put('$initial_call', MFA), put(nonce, Nonce), try + csrt:create_worker_context(From, MFA, Nonce), + couch_stats:maybe_track_rexi_init_p(MFA), apply(M, F, A) catch exit:normal -> + csrt:destroy_context(), ok; Class:Reason:Stack0 -> + csrt:destroy_context(), Stack = clean_stack(Stack0), {ClientPid, _ClientRef} = From, couch_log:error( @@ -158,6 +163,7 @@ init_p(From, {M, F, A}, Nonce) -> ] ), exit(#error{ + delta = csrt:make_delta(), timestamp = os:timestamp(), reason = {Class, Reason}, mfa = {M, F, A}, @@ -200,8 +206,9 @@ find_worker(Ref, Tab) -> [Worker] -> Worker end. -notify_caller({Caller, Ref}, Reason) -> - rexi_utils:send(Caller, {Ref, {rexi_EXIT, Reason}}). +notify_caller({Caller, Ref}, Reason, Delta) -> + Msg = csrt:maybe_add_delta({Ref, {rexi_EXIT, Reason}}, Delta), + rexi_utils:send(Caller, Msg). kill_worker(FromRef, #st{clients = Clients} = St) -> case find_worker(FromRef, Clients) of diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl index 146d0238ac1..4ee2586ec7d 100644 --- a/src/rexi/src/rexi_utils.erl +++ b/src/rexi/src/rexi_utils.erl @@ -60,6 +60,16 @@ process_mailbox(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> receive + Msg -> + process_raw_message(Msg, RefList, Keypos, Fun, Acc0, TimeoutRef) + after PerMsgTO -> + {timeout, Acc0} + end. + +process_raw_message(Payload0, RefList, Keypos, Fun, Acc0, TimeoutRef) -> + {Payload, Delta} = csrt:extract_delta(Payload0), + csrt:accumulate_delta(Delta), + case Payload of {timeout, TimeoutRef} -> {timeout, Acc0}; {rexi, Ref, Msg} -> @@ -95,6 +105,4 @@ process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> end; {rexi_DOWN, _, _, _} = Msg -> Fun(Msg, nil, Acc0) - after PerMsgTO -> - {timeout, Acc0} end. diff --git a/src/rexi/test/rexi_tests.erl b/src/rexi/test/rexi_tests.erl index 18b05b545ca..6a388d16386 100644 --- a/src/rexi/test/rexi_tests.erl +++ b/src/rexi/test/rexi_tests.erl @@ -75,6 +75,7 @@ t_cast(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [potato]}), {Res, Dict} = receive + {Ref, {R, D}, {delta, _}} -> {R, maps:from_list(D)}; {Ref, {R, D}} -> {R, maps:from_list(D)} end, ?assertEqual(potato, Res), @@ -99,7 +100,12 @@ t_cast_explicit_caller(_) -> receive {'DOWN', CallerRef, _, _, Exit} -> Exit end, - ?assertMatch({Ref, {potato, [_ | _]}}, Result). + case csrt:is_enabled() of + true -> + ?assertMatch({Ref, {potato, [_ | _]}, {delta, _}}, Result); + false -> + ?assertMatch({Ref, {potato, [_ | _]}}, Result) + end. t_cast_ref(_) -> put(nonce, yesh), @@ -180,6 +186,7 @@ t_cast_error(_) -> Ref = rexi:cast(node(), self(), {?MODULE, rpc_test_fun, [{error, tomato}]}, []), Res = receive + {Ref, RexiExit, {delta, _}} -> RexiExit; {Ref, RexiExit} -> RexiExit end, ?assertMatch({rexi_EXIT, {tomato, [{?MODULE, rpc_test_fun, 1, _} | _]}}, Res). @@ -188,6 +195,7 @@ t_kill(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [{sleep, 10000}]}), WorkerPid = receive + {Ref, {sleeping, Pid}, {delta, _}} -> Pid; {Ref, {sleeping, Pid}} -> Pid end, ?assert(is_process_alive(WorkerPid)), @@ -207,18 +215,23 @@ t_ping(_) -> rexi:cast(node(), {?MODULE, rpc_test_fun, [ping]}), Res = receive + {rexi, Ping, {delta, _}} -> Ping; {rexi, Ping} -> Ping end, ?assertEqual('$rexi_ping', Res). stream_init(Ref) -> receive + {Ref, From, rexi_STREAM_INIT, {delta, _}} -> + From; {Ref, From, rexi_STREAM_INIT} -> From end. recv(Ref) when is_reference(Ref) -> receive + {Ref, _, Msg, {delta, _}} -> Msg; + {Ref, Msg, {delta, _}} -> Msg; {Ref, _, Msg} -> Msg; {Ref, Msg} -> Msg after 500 -> timeout From d99c8127704a2e16b4e54222a2156b5d44f45e96 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 25 Mar 2025 15:58:41 -0700 Subject: [PATCH 02/54] Remove no longer used conf_get fun --- src/couch_stats/src/csrt.erl | 14 +------------- src/couch_stats/src/csrt_util.erl | 10 ---------- 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index a2b5f51d85c..d43fde66fe3 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -47,9 +47,7 @@ is_enabled/0, is_enabled_init_p/0, do_report/2, - maybe_report/2, - conf_get/1, - conf_get/2 + maybe_report/2 ]). %% stats collection api @@ -345,16 +343,6 @@ rctx_delta(TA, TB) -> %%update_counter(Field, Count) when Count >= 0 -> %% is_enabled() andalso csrt_server:update_counter(get_pid_ref(), Field, Count). - --spec conf_get(Key :: string()) -> string(). -conf_get(Key) -> - csrt_util:conf_get(Key). - - --spec conf_get(Key :: string(), Default :: string()) -> string(). -conf_get(Key, Default) -> - csrt_util:conf_get(Key, Default). - %% %% aggregate query api %% diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index c386cf08ee0..d108ffffc84 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -15,8 +15,6 @@ -export([ is_enabled/0, is_enabled_init_p/0, - conf_get/1, - conf_get/2, get_pid_ref/0, get_pid_ref/1, set_pid_ref/1, @@ -77,14 +75,6 @@ should_track_init_p(fabric_rpc, Func) -> should_track_init_p(_Mod, _Func) -> false. --spec conf_get(Key :: list()) -> list(). -conf_get(Key) -> - conf_get(Key, undefined). - --spec conf_get(Key :: list(), Default :: list()) -> list(). -conf_get(Key, Default) -> - config:get(?CSRT, Key, Default). - %% Monotnonic time now in native format using time forward only event tracking -spec tnow() -> integer(). tnow() -> From 922b3f84bf96eb87bb9157a0f33128119fe994fb Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 25 Mar 2025 16:03:09 -0700 Subject: [PATCH 03/54] Cleanup Dialyzer specs --- src/couch_stats/src/couch_stats_resource_tracker.hrl | 2 +- src/couch_stats/src/csrt_logger.erl | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 1fd1a99a141..d47c0055be8 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -167,5 +167,5 @@ -type matcher_name() :: string(). %% TODO: switch to string to allow dynamic options -type matcher() :: {ets:match_spec(), ets:comp_match_spec()}. -type maybe_matcher() :: matcher() | undefined. --type matchers() :: #{matcher_name() => matcher()}. +-type matchers() :: #{matcher_name() => matcher()} | #{}. -type maybe_matchers() :: matchers() | undefined. diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 2e692e88836..073e2727e40 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -119,7 +119,7 @@ log_process_lifetime_report(PidRef) -> end. %% TODO: add Matchers spec --spec find_matches(Rctxs :: [rctx()], Matchers :: [any()]) -> matchers(). +-spec find_matches(Rctxs :: [rctx()], Matchers :: matchers()) -> matchers(). find_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> maps:filter( fun(_Name, {_MSpec, CompMSpec}) -> @@ -147,7 +147,7 @@ is_match(#rctx{}=Rctx) -> is_match(Rctx, get_matchers()). %% TODO: add Matchers spec --spec is_match(Rctx :: maybe_rctx(), Matchers :: [any()]) -> boolean(). +-spec is_match(Rctx :: maybe_rctx(), Matchers :: matchers()) -> boolean(). is_match(undefined, _Matchers) -> false; is_match(_Rctx, undefined) -> @@ -301,7 +301,7 @@ add_matcher(Name, MSpec, Matchers) -> {error, badarg} end. --spec set_matchers_term(Matchers :: matchers()) -> maybe_matchers(). +-spec set_matchers_term(Matchers :: matchers()) -> ok. set_matchers_term(Matchers) when is_map(Matchers) -> persistent_term:put({?MODULE, all_csrt_matchers}, Matchers). From 375ec28993beb4a04c4150e32ba45d3af5dfe2f9 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 7 Apr 2025 21:51:58 -0700 Subject: [PATCH 04/54] Fix type in metric name --- src/couch/src/couch_os_process.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 003c3dc519d..637c3d33634 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -260,7 +260,7 @@ bump_time_stat(Stat, USec) when is_atom(Stat), is_integer(USec) -> couch_stats:increment_counter([couchdb, query_server, time, Stat], USec). bump_volume_stat(ddoc_filter=Stat, Docs) when is_atom(Stat), is_list(Docs) -> - couch_stats:increment_counter([couchdb, query_server, volume, Stat, length(Docs)]); + couch_stats:increment_counter([couchdb, query_server, volume, Stat], length(Docs)); bump_volume_stat(_, _) -> %% TODO: handle other stats? ok. From ada453ed72efce8d8aba0341d5fcbc33cbc002c9 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 7 Apr 2025 21:56:24 -0700 Subject: [PATCH 05/54] Update CSRT tests for ioq parallel read changes --- src/couch_stats/test/eunit/csrt_logger_tests.erl | 3 +++ src/couch_stats/test/eunit/csrt_server_tests.erl | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 648b7601cfe..eaadab70c9c 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -72,6 +72,8 @@ make_docs(Count) -> setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), + ok = meck:new(ioq, [passthrough]), + ok = meck:expect(ioq, bypass, fun(_, _) -> false end), DbName = ?tempdb(), ok = fabric:create_db(DbName, [{q, 8}, {n, 1}]), Docs = make_docs(100), @@ -97,6 +99,7 @@ setup() -> teardown(#{ctx := Ctx, dbname := DbName}) -> ok = fabric:delete_db(DbName, [?ADMIN_CTX]), + ok = meck:unload(ioq), test_util:stop_couch(Ctx). rctx_gen() -> diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index f3cf07a836a..e4534b8edcb 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -85,6 +85,8 @@ make_docs(Count) -> setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), + ok = meck:new(ioq, [passthrough]), + ok = meck:expect(ioq, bypass, fun(_, _) -> false end), DbName = ?tempdb(), ok = fabric:create_db(DbName, [{q, ?DB_Q}, {n, 1}]), Docs = make_docs(?DOCS_COUNT), @@ -94,6 +96,7 @@ setup() -> teardown({Ctx, DbName, _View}) -> ok = fabric:delete_db(DbName, [?ADMIN_CTX]), + ok = meck:unload(ioq), test_util:stop_couch(Ctx). setup_ddoc(DDocId, ViewName) -> From 9aadac494baf0d1f7f2dfe50ee161aadd5d72ff3 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 7 Apr 2025 21:58:40 -0700 Subject: [PATCH 06/54] Add csrt_logger:register_matcher --- src/couch_stats/src/csrt_logger.erl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 073e2727e40..86e63f489d6 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -59,6 +59,7 @@ matcher_on_worker_changes_processed/1, matcher_on_ioq_calls/1, matcher_on_nonce/1, + register_matcher/2, reload_matchers/0 ]). @@ -109,6 +110,11 @@ tracker({Pid, _Ref}=PidRef) -> ok end. +-spec register_matcher(Name, MSpec) -> ok | {error, badarg} when + Name :: string(), MSpec :: ets:match_spec(). +register_matcher(Name, MSpec) -> + gen_server:call(?MODULE, {register, Name, MSpec}). + -spec log_process_lifetime_report(PidRef :: pid_ref()) -> ok. log_process_lifetime_report(PidRef) -> case csrt_util:is_enabled() of From 1421a5000aec567cb5c1b4b1df180dc4ac167932 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 7 Apr 2025 22:12:00 -0700 Subject: [PATCH 07/54] Rework changes_processed vs rows --- .../src/couch_stats_resource_tracker.hrl | 14 +++++++------- src/couch_stats/test/eunit/csrt_server_tests.erl | 14 ++++---------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index d47c0055be8..82a7ead7ba9 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -32,24 +32,24 @@ -define(IOQ_CALLS, ioq_calls). -define(DOCS_WRITTEN, docs_written). -define(ROWS_READ, rows_read). - -%% TODO: overlap between this and couch btree fold invocations -%% TODO: need some way to distinguish fols on views vs find vs all_docs --define(FRPC_CHANGES_ROW, changes_processed). +%% TODO: use dedicated changes_processed or use rows_read? +%%-define(FRPC_CHANGES_PROCESSED, rows_read). +-define(FRPC_CHANGES_PROCESSED, changes_processed). -define(FRPC_CHANGES_RETURNED, changes_returned). -define(STATS_TO_KEYS, #{ [mango, evaluate_selector] => ?MANGO_EVAL_MATCH, [couchdb, database_reads] => ?DB_OPEN_DOC, - [fabric_rpc, changes, processed] => ?FRPC_CHANGES_ROW, + [fabric_rpc, changes, processed] => ?FRPC_CHANGES_PROCESSED, [fabric_rpc, changes, returned] => ?FRPC_CHANGES_RETURNED, [fabric_rpc, view, rows_read] => ?ROWS_READ, [couchdb, couch_server, open] => ?DB_OPEN, [couchdb, btree, get_node, kp_node] => ?COUCH_BT_GET_KP_NODE, [couchdb, btree, get_node, kv_node] => ?COUCH_BT_GET_KV_NODE, + + %% NOTE: these stats are not local to the RPC worker, need forwarding [couchdb, btree, write_node, kp_node] => ?COUCH_BT_WRITE_KP_NODE, [couchdb, btree, write_node, kv_node] => ?COUCH_BT_WRITE_KV_NODE, - %% NOTE: these stats are not local to the RPC worker, need forwarding [couchdb, query_server, calls, ddoc_filter] => ?COUCH_JS_FILTER, [couchdb, query_server, volume, ddoc_filter] => ?COUCH_JS_FILTERED_DOCS }). @@ -58,13 +58,13 @@ ?DB_OPEN => #rctx.?DB_OPEN, ?ROWS_READ => #rctx.?ROWS_READ, ?FRPC_CHANGES_RETURNED => #rctx.?FRPC_CHANGES_RETURNED, + ?FRPC_CHANGES_PROCESSED => #rctx.?FRPC_CHANGES_PROCESSED, ?DOCS_WRITTEN => #rctx.?DOCS_WRITTEN, ?IOQ_CALLS => #rctx.?IOQ_CALLS, ?COUCH_JS_FILTER => #rctx.?COUCH_JS_FILTER, ?COUCH_JS_FILTERED_DOCS => #rctx.?COUCH_JS_FILTERED_DOCS, ?MANGO_EVAL_MATCH => #rctx.?MANGO_EVAL_MATCH, ?DB_OPEN_DOC => #rctx.?DB_OPEN_DOC, - ?FRPC_CHANGES_ROW => #rctx.?ROWS_READ, %% TODO: rework double use of rows_read ?COUCH_BT_GET_KP_NODE => #rctx.?COUCH_BT_GET_KP_NODE, ?COUCH_BT_GET_KV_NODE => #rctx.?COUCH_BT_GET_KV_NODE, ?COUCH_BT_WRITE_KP_NODE => #rctx.?COUCH_BT_WRITE_KP_NODE, diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index e4534b8edcb..e26dfc7cc32 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -311,16 +311,12 @@ t_changes({_Ctx, DbName, View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => ?DB_Q, - rows_read => assert_gte(?DB_Q), + changes_processed => docs_count(View), changes_returned => docs_count(View), docs_read => 0, docs_written => 0, pid_ref => PidRef }), - %% at least one rows_read and changes_returned per shard that has at least - %% one document in it - ?assert(maps:get(rows_read, Rctx) >= ?DB_Q, rows_read), - ?assert(maps:get(changes_returned, Rctx) >= ?DB_Q, changes_returned), ok = nonzero_local_io_assert(Rctx), ok = assert_teardown(PidRef). @@ -338,14 +334,12 @@ t_changes_limit_zero({_Ctx, DbName, _View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => ?DB_Q, - rows_read => false, - changes_returned => false, + changes_processed => assert_gte(?DB_Q), + changes_returned => assert_gte(?DB_Q), docs_read => 0, docs_written => 0, pid_ref => PidRef }), - ?assert(maps:get(rows_read, Rctx) >= ?DB_Q, rows), - ?assert(maps:get(changes_returned, Rctx) >= ?DB_Q, rows), ok = nonzero_local_io_assert(Rctx), ok = assert_teardown(PidRef). @@ -372,7 +366,7 @@ t_changes_js_filtered({_Ctx, DbName, {DDocId, _ViewName}=View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => assert_gte(?DB_Q), - rows_read => assert_gte(docs_count(View)), + changes_processed => assert_gte(docs_count(View)), changes_returned => round(?DOCS_COUNT / 2), docs_read => assert_gte(docs_count(View)), docs_written => 0, From cba2e9cf1baf83874589d7513a07e0ff29f1067d Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 7 Apr 2025 22:21:36 -0700 Subject: [PATCH 08/54] Format code --- src/chttpd/src/chttpd_misc.erl | 108 ++++++++--------- src/couch/src/couch_os_process.erl | 2 +- src/couch_stats/src/csrt.erl | 25 ++-- src/couch_stats/src/csrt_logger.erl | 111 +++++++++++------- src/couch_stats/src/csrt_query.erl | 47 ++++---- src/couch_stats/src/csrt_server.erl | 34 +++--- src/couch_stats/src/csrt_util.erl | 15 ++- .../test/eunit/csrt_logger_tests.erl | 86 ++++++++------ .../test/eunit/csrt_server_tests.erl | 59 +++++----- src/fabric/test/eunit/fabric_rpc_tests.erl | 9 +- 10 files changed, 276 insertions(+), 220 deletions(-) diff --git a/src/chttpd/src/chttpd_misc.erl b/src/chttpd/src/chttpd_misc.erl index 0baf0b972e3..415aeab20d9 100644 --- a/src/chttpd/src/chttpd_misc.erl +++ b/src/chttpd/src/chttpd_misc.erl @@ -237,65 +237,68 @@ handle_resource_status_req(#httpd{method = 'POST'} = Req) -> ToJson = fun csrt_util:to_json/1, JsonKeys = fun(PL) -> [[ToJson(K), V] || {K, V} <- PL] end, - Fun = case {Action, Key, Val} of - {<<"count_by">>, Keys, undefined} when is_list(Keys) -> - Keys1 = [ConvertEle(K) || K <- Keys], - fun() -> CountBy(Keys1) end; - {<<"count_by">>, Key, undefined} -> - Key1 = ConvertEle(Key), - fun() -> CountBy(Key1) end; - {<<"group_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> - Keys1 = ConvertList(Keys), - Vals1 = ConvertList(Vals), - fun() -> GroupBy(Keys1, Vals1) end; - {<<"group_by">>, Key, Vals} when is_list(Vals) -> - Key1 = ConvertEle(Key), - Vals1 = ConvertList(Vals), - fun() -> GroupBy(Key1, Vals1) end; - {<<"group_by">>, Keys, Val} when is_list(Keys) -> - Keys1 = ConvertList(Keys), - Val1 = ConvertEle(Val), - fun() -> GroupBy(Keys1, Val1) end; - {<<"group_by">>, Key, Val} -> - Key1 = ConvertEle(Key), - Val1 = ConvertList(Val), - fun() -> GroupBy(Key1, Val1) end; - - {<<"sorted_by">>, Key, undefined} -> - Key1 = ConvertEle(Key), - fun() -> JsonKeys(SortedBy1(Key1)) end; - {<<"sorted_by">>, Keys, undefined} when is_list(Keys) -> - Keys1 = [ConvertEle(K) || K <- Keys], - fun() -> JsonKeys(SortedBy1(Keys1)) end; - {<<"sorted_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> - Keys1 = ConvertList(Keys), - Vals1 = ConvertList(Vals), - fun() -> JsonKeys(SortedBy2(Keys1, Vals1)) end; - {<<"sorted_by">>, Key, Vals} when is_list(Vals) -> - Key1 = ConvertEle(Key), - Vals1 = ConvertList(Vals), - fun() -> JsonKeys(SortedBy2(Key1, Vals1)) end; - {<<"sorted_by">>, Keys, Val} when is_list(Keys) -> - Keys1 = ConvertList(Keys), - Val1 = ConvertEle(Val), - fun() -> JsonKeys(SortedBy2(Keys1, Val1)) end; - {<<"sorted_by">>, Key, Val} -> - Key1 = ConvertEle(Key), - Val1 = ConvertList(Val), - fun() -> JsonKeys(SortedBy2(Key1, Val1)) end; - _ -> - throw({badrequest, invalid_resource_request}) - end, + Fun = + case {Action, Key, Val} of + {<<"count_by">>, Keys, undefined} when is_list(Keys) -> + Keys1 = [ConvertEle(K) || K <- Keys], + fun() -> CountBy(Keys1) end; + {<<"count_by">>, Key, undefined} -> + Key1 = ConvertEle(Key), + fun() -> CountBy(Key1) end; + {<<"group_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> + Keys1 = ConvertList(Keys), + Vals1 = ConvertList(Vals), + fun() -> GroupBy(Keys1, Vals1) end; + {<<"group_by">>, Key, Vals} when is_list(Vals) -> + Key1 = ConvertEle(Key), + Vals1 = ConvertList(Vals), + fun() -> GroupBy(Key1, Vals1) end; + {<<"group_by">>, Keys, Val} when is_list(Keys) -> + Keys1 = ConvertList(Keys), + Val1 = ConvertEle(Val), + fun() -> GroupBy(Keys1, Val1) end; + {<<"group_by">>, Key, Val} -> + Key1 = ConvertEle(Key), + Val1 = ConvertList(Val), + fun() -> GroupBy(Key1, Val1) end; + {<<"sorted_by">>, Key, undefined} -> + Key1 = ConvertEle(Key), + fun() -> JsonKeys(SortedBy1(Key1)) end; + {<<"sorted_by">>, Keys, undefined} when is_list(Keys) -> + Keys1 = [ConvertEle(K) || K <- Keys], + fun() -> JsonKeys(SortedBy1(Keys1)) end; + {<<"sorted_by">>, Keys, Vals} when is_list(Keys) andalso is_list(Vals) -> + Keys1 = ConvertList(Keys), + Vals1 = ConvertList(Vals), + fun() -> JsonKeys(SortedBy2(Keys1, Vals1)) end; + {<<"sorted_by">>, Key, Vals} when is_list(Vals) -> + Key1 = ConvertEle(Key), + Vals1 = ConvertList(Vals), + fun() -> JsonKeys(SortedBy2(Key1, Vals1)) end; + {<<"sorted_by">>, Keys, Val} when is_list(Keys) -> + Keys1 = ConvertList(Keys), + Val1 = ConvertEle(Val), + fun() -> JsonKeys(SortedBy2(Keys1, Val1)) end; + {<<"sorted_by">>, Key, Val} -> + Key1 = ConvertEle(Key), + Val1 = ConvertList(Val), + fun() -> JsonKeys(SortedBy2(Key1, Val1)) end; + _ -> + throw({badrequest, invalid_resource_request}) + end, Fun1 = fun() -> case Fun() of Map when is_map(Map) -> {maps:fold( fun - (_K,0,A) -> A; %% TODO: Skip 0 value entries? - (K,V,A) -> [{ToJson(K), V} | A] + %% TODO: Skip 0 value entries? + (_K, 0, A) -> A; + (K, V, A) -> [{ToJson(K), V} | A] end, - [], Map)}; + [], + Map + )}; List when is_list(List) -> List end @@ -323,7 +326,6 @@ handle_resource_status_req(Req) -> ok = chttpd:verify_is_server_admin(Req), send_method_not_allowed(Req, "GET,HEAD,POST"). - handle_replicate_req(#httpd{method = 'POST', user_ctx = Ctx, req_body = PostBody} = Req) -> chttpd:validate_ctype(Req, "application/json"), %% see HACK in chttpd.erl about replication diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 637c3d33634..1a018bd0417 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -259,7 +259,7 @@ bump_time_stat(Stat, USec) when is_atom(Stat), is_integer(USec) -> couch_stats:increment_counter([couchdb, query_server, calls, Stat]), couch_stats:increment_counter([couchdb, query_server, time, Stat], USec). -bump_volume_stat(ddoc_filter=Stat, Docs) when is_atom(Stat), is_list(Docs) -> +bump_volume_stat(ddoc_filter = Stat, Docs) when is_atom(Stat), is_list(Docs) -> couch_stats:increment_counter([couchdb, query_server, volume, Stat], length(Docs)); bump_volume_stat(_, _) -> %% TODO: handle other stats? diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index d43fde66fe3..2d8dc8670af 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -130,22 +130,22 @@ destroy_pid_ref(_PidRef) -> -spec create_worker_context(From, MFA, Nonce) -> pid_ref() | false when From :: pid_ref(), MFA :: mfa(), Nonce :: term(). -create_worker_context(From, {M,F,_A}, Nonce) -> +create_worker_context(From, {M, F, _A}, Nonce) -> case is_enabled() of true -> - Type = #rpc_worker{from=From, mod=M, func=F}, + Type = #rpc_worker{from = From, mod = M, func = F}, create_context(Type, Nonce); false -> false end. --spec create_coordinator_context(Httpd , Path) -> pid_ref() | false when +-spec create_coordinator_context(Httpd, Path) -> pid_ref() | false when Httpd :: #httpd{}, Path :: list(). -create_coordinator_context(#httpd{method=Verb, nonce=Nonce}, Path0) -> +create_coordinator_context(#httpd{method = Verb, nonce = Nonce}, Path0) -> case is_enabled() of true -> Path = list_to_binary([$/ | Path0]), - Type = #coordinator{method=Verb, path=Path}, + Type = #coordinator{method = Verb, path = Path}, create_context(Type, Nonce); false -> false @@ -188,8 +188,9 @@ set_context_handler_fun(Fun) when is_function(Fun) -> end. -spec set_context_handler_fun(Mod :: atom(), Func :: atom()) -> boolean(). -set_context_handler_fun(Mod, Func) - when is_atom(Mod) andalso is_atom(Func) -> +set_context_handler_fun(Mod, Func) when + is_atom(Mod) andalso is_atom(Func) +-> case is_enabled() of false -> false; @@ -205,7 +206,7 @@ update_handler_fun(Mod, Func, PidRef) -> Rctx = get_resource(PidRef), %% TODO: #coordinator{} assumption needs to adapt for other types #coordinator{} = Coordinator0 = csrt_server:get_context_type(Rctx), - Coordinator = Coordinator0#coordinator{mod=Mod, func=Func}, + Coordinator = Coordinator0#coordinator{mod = Mod, func = Func}, csrt_server:set_context_type(Coordinator, PidRef), ok. @@ -283,7 +284,6 @@ inc(Key) -> inc(Key, N) when is_integer(N) andalso N >= 0 -> is_enabled() andalso csrt_server:inc(get_pid_ref(), Key, N). - -spec maybe_inc(Stat :: atom(), Val :: non_neg_integer()) -> non_neg_integer(). maybe_inc(Stat, Val) -> case maps:is_key(Stat, ?STATS_TO_KEYS) of @@ -418,7 +418,6 @@ add_delta(T, Delta) -> extract_delta(T) -> csrt_util:extract_delta(T). - get_delta() -> csrt_util:get_delta(get_pid_ref()). @@ -432,7 +431,6 @@ maybe_add_delta(T, Delta) -> %% Internal Operations assuming is_enabled() == true %% - -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). @@ -458,7 +456,10 @@ teardown(Ctx) -> t_static_map_translations(_) -> ?assert(lists:all(fun(E) -> maps:is_key(E, ?KEYS_TO_FIELDS) end, maps:values(?STATS_TO_KEYS))), %% TODO: properly handle ioq_calls field - ?assertEqual(lists:sort(maps:values(?STATS_TO_KEYS)), lists:delete(docs_written, lists:delete(ioq_calls, lists:sort(maps:keys(?KEYS_TO_FIELDS))))). + ?assertEqual( + lists:sort(maps:values(?STATS_TO_KEYS)), + lists:delete(docs_written, lists:delete(ioq_calls, lists:sort(maps:keys(?KEYS_TO_FIELDS)))) + ). t_should_track_init_p(_) -> config:set(?CSRT_INIT_P, "enabled", "true", false), diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 86e63f489d6..b31e18afd3d 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -76,7 +76,7 @@ }). -spec track(Rctx :: rctx()) -> pid(). -track(#rctx{pid_ref=PidRef}) -> +track(#rctx{pid_ref = PidRef}) -> case get_tracker() of undefined -> Pid = spawn(?MODULE, tracker, [PidRef]), @@ -87,7 +87,7 @@ track(#rctx{pid_ref=PidRef}) -> end. -spec tracker(PidRef :: pid_ref()) -> ok. -tracker({Pid, _Ref}=PidRef) -> +tracker({Pid, _Ref} = PidRef) -> MonRef = erlang:monitor(process, Pid), receive stop -> @@ -111,7 +111,7 @@ tracker({Pid, _Ref}=PidRef) -> end. -spec register_matcher(Name, MSpec) -> ok | {error, badarg} when - Name :: string(), MSpec :: ets:match_spec(). + Name :: string(), MSpec :: ets:match_spec(). register_matcher(Name, MSpec) -> gen_server:call(?MODULE, {register, Name, MSpec}). @@ -149,7 +149,7 @@ get_matcher(Name) -> -spec is_match(Rctx :: maybe_rctx()) -> boolean(). is_match(undefined) -> false; -is_match(#rctx{}=Rctx) -> +is_match(#rctx{} = Rctx) -> is_match(Rctx, get_matchers()). %% TODO: add Matchers spec @@ -158,7 +158,7 @@ is_match(undefined, _Matchers) -> false; is_match(_Rctx, undefined) -> false; -is_match(#rctx{}=Rctx, Matchers) when is_map(Matchers) -> +is_match(#rctx{} = Rctx, Matchers) when is_map(Matchers) -> maps:size(find_matches([Rctx], Matchers)) > 0. -spec maybe_report(ReportName :: string(), PidRef :: maybe_pid_ref()) -> ok. @@ -181,7 +181,7 @@ do_status_report(Rctx) -> do_report("csrt-pid-usage-status", Rctx). -spec do_report(ReportName :: string(), Rctx :: rctx()) -> boolean(). -do_report(ReportName, #rctx{}=Rctx) -> +do_report(ReportName, #rctx{} = Rctx) -> couch_log:report(ReportName, csrt_util:to_json(Rctx)). %% @@ -215,12 +215,12 @@ init([]) -> ok = subscribe_changes(), {ok, #st{}}. -handle_call({register, Name, MSpec}, _From, #st{matchers=Matchers}=St) -> +handle_call({register, Name, MSpec}, _From, #st{matchers = Matchers} = St) -> case add_matcher(Name, MSpec, Matchers) of {ok, Matchers1} -> set_matchers_term(Matchers1), - {reply, ok, St#st{matchers=Matchers1}}; - {error, badarg}=Error -> + {reply, ok, St#st{matchers = Matchers1}}; + {error, badarg} = Error -> {reply, Error, St} end; handle_call(reload_matchers, _From, St) -> @@ -244,45 +244,67 @@ handle_info(_Msg, St) -> %% -spec matcher_on_dbname(DbName :: dbname()) -> ets:match_spec(). -matcher_on_dbname(DbName) - when is_binary(DbName) -> - ets:fun2ms(fun(#rctx{dbname=DbName1} = R) when DbName =:= DbName1 -> R end). +matcher_on_dbname(DbName) when + is_binary(DbName) +-> + ets:fun2ms(fun(#rctx{dbname = DbName1} = R) when DbName =:= DbName1 -> R end). -spec matcher_on_dbname_io_threshold(DbName, Threshold) -> ets:match_spec() when - DbName :: dbname(), Threshold :: pos_integer(). -matcher_on_dbname_io_threshold(DbName, Threshold) - when is_binary(DbName) -> - ets:fun2ms(fun(#rctx{dbname=DbName1, ioq_calls=IOQ, get_kv_node=KVN, get_kp_node=KPN, docs_read=Docs, rows_read=Rows, changes_processed=Chgs} = R) when DbName =:= DbName1 andalso ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or (Rows >= Threshold) or (Chgs >= Threshold)) -> R end). + DbName :: dbname(), Threshold :: pos_integer(). +matcher_on_dbname_io_threshold(DbName, Threshold) when + is_binary(DbName) +-> + ets:fun2ms(fun( + #rctx{ + dbname = DbName1, + ioq_calls = IOQ, + get_kv_node = KVN, + get_kp_node = KPN, + docs_read = Docs, + rows_read = Rows, + changes_processed = Chgs + } = R + ) when + DbName =:= DbName1 andalso + ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or + (Rows >= Threshold) or (Chgs >= Threshold)) + -> + R + end). -spec matcher_on_docs_read(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_docs_read(Threshold) - when is_integer(Threshold) andalso Threshold > 0 -> +matcher_on_docs_read(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> %%ets:fun2ms(fun(#rctx{type=#coordinator{}, docs_read=DocsRead} = R) when DocsRead >= Threshold -> R end). - ets:fun2ms(fun(#rctx{docs_read=DocsRead} = R) when DocsRead >= Threshold -> R end). + ets:fun2ms(fun(#rctx{docs_read = DocsRead} = R) when DocsRead >= Threshold -> R end). -spec matcher_on_docs_written(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_docs_written(Threshold) - when is_integer(Threshold) andalso Threshold > 0 -> +matcher_on_docs_written(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> %%ets:fun2ms(fun(#rctx{type=#coordinator{}, docs_written=DocsRead} = R) when DocsRead >= Threshold -> R end). - ets:fun2ms(fun(#rctx{docs_written=DocsWritten} = R) when DocsWritten >= Threshold -> R end). + ets:fun2ms(fun(#rctx{docs_written = DocsWritten} = R) when DocsWritten >= Threshold -> R end). -spec matcher_on_rows_read(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_rows_read(Threshold) - when is_integer(Threshold) andalso Threshold > 0 -> - ets:fun2ms(fun(#rctx{rows_read=RowsRead} = R) when RowsRead >= Threshold -> R end). +matcher_on_rows_read(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> + ets:fun2ms(fun(#rctx{rows_read = RowsRead} = R) when RowsRead >= Threshold -> R end). -spec matcher_on_nonce(Nonce :: nonce()) -> ets:match_spec(). matcher_on_nonce(Nonce) -> ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end). -spec matcher_on_worker_changes_processed(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_worker_changes_processed(Threshold) - when is_integer(Threshold) andalso Threshold > 0 -> +matcher_on_worker_changes_processed(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> ets:fun2ms( fun( #rctx{ - changes_processed=Processed, - changes_returned=Returned + changes_processed = Processed, + changes_returned = Returned } = R ) when (Processed - Returned) >= Threshold -> R @@ -290,12 +312,13 @@ matcher_on_worker_changes_processed(Threshold) ). -spec matcher_on_ioq_calls(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_ioq_calls(Threshold) - when is_integer(Threshold) andalso Threshold > 0 -> - ets:fun2ms(fun(#rctx{ioq_calls=IOQCalls} = R) when IOQCalls >= Threshold -> R end). +matcher_on_ioq_calls(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> + ets:fun2ms(fun(#rctx{ioq_calls = IOQCalls} = R) when IOQCalls >= Threshold -> R end). -spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, badarg} when - Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). + Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). add_matcher(Name, MSpec, Matchers) -> try ets:match_spec_compile(MSpec) of CompMSpec -> @@ -334,7 +357,9 @@ initialize_matchers() -> {ok, Matchers1} -> Matchers1; {error, badarg} -> - couch_log:warning("[~p] Failed to initialize matcher: ~p", [?MODULE, Name]), + couch_log:warning("[~p] Failed to initialize matcher: ~p", [ + ?MODULE, Name + ]), Matchers0 end; false -> @@ -355,13 +380,18 @@ initialize_matchers() -> {ok, Matchers1} -> Matchers1; {error, badarg} -> - couch_log:warning("[~p] Failed to initialize matcher: ~p", [?MODULE, Name]), + couch_log:warning("[~p] Failed to initialize matcher: ~p", [ + ?MODULE, Name + ]), Matchers0 end; _ -> Matchers0 - catch error:badarg -> - couch_log:warning("[~p] Failed to initialize dbname io matcher on: ~p", [?MODULE, Dbname]) + catch + error:badarg -> + couch_log:warning("[~p] Failed to initialize dbname io matcher on: ~p", [ + ?MODULE, Dbname + ]) end end, Matchers, @@ -379,14 +409,15 @@ matcher_enabled(Name) when is_list(Name) -> config:get_boolean(?CONF_MATCHERS_ENABLED, Name, true). -spec matcher_threshold(Name, Threshold) -> string() | integer() when - Name :: string(), Threshold :: pos_integer() | string(). + Name :: string(), Threshold :: pos_integer() | string(). matcher_threshold("dbname", DbName) when is_binary(DbName) -> %% TODO: toggle Default to undefined to disallow for particular dbname %% TODO: sort out list vs binary %%config:get_integer(?CONF_MATCHERS_THRESHOLD, binary_to_list(DbName), Default); DbName; -matcher_threshold(Name, Default) - when is_list(Name) andalso is_integer(Default) andalso Default > 0 -> +matcher_threshold(Name, Default) when + is_list(Name) andalso is_integer(Default) andalso Default > 0 +-> config:get_integer(?CONF_MATCHERS_THRESHOLD, Name, Default). subscribe_changes() -> diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl index a3580c58a8d..3de8f417bb1 100644 --- a/src/couch_stats/src/csrt_query.erl +++ b/src/couch_stats/src/csrt_query.erl @@ -75,48 +75,48 @@ select_by_type(all) -> find_by_nonce(Nonce) -> %%ets:match_object(?MODULE, ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end)). - [R || R <- ets:match_object(?MODULE, #rctx{nonce=Nonce})]. + [R || R <- ets:match_object(?MODULE, #rctx{nonce = Nonce})]. find_by_pid(Pid) -> %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{pid_ref={Pid, '_'}, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{pid_ref={Pid, '_'}})]. + [R || R <- ets:match_object(?MODULE, #rctx{pid_ref = {Pid, '_'}})]. find_by_pidref(PidRef) -> %%[R || R <- ets:match_object(?MODULE, #rctx{pid_ref=PidRef, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{pid_ref=PidRef})]. + [R || R <- ets:match_object(?MODULE, #rctx{pid_ref = PidRef})]. find_workers_by_pidref(PidRef) -> %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{type=#rpc_worker{from=PidRef}, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{type=#rpc_worker{from=PidRef}})]. + [R || R <- ets:match_object(?MODULE, #rctx{type = #rpc_worker{from = PidRef}})]. -field(#rctx{pid_ref=Val}, pid_ref) -> Val; +field(#rctx{pid_ref = Val}, pid_ref) -> Val; %% NOTE: Pros and cons to doing these convert functions here %% Ideally, this would be done later so as to prefer the core data structures %% as long as possible, but we currently need the output of this function to %% be jiffy:encode'able. The tricky bit is dynamically encoding the group_by %% structure provided by the caller of *_by aggregator functions below. %% For now, we just always return jiffy:encode'able data types. -field(#rctx{nonce=Val}, nonce) -> Val; +field(#rctx{nonce = Val}, nonce) -> Val; %%field(#rctx{from=Val}, from) -> Val; %% TODO: fix this, perhaps move it all to csrt_util? -field(#rctx{type=Val}, type) -> csrt_util:convert_type(Val); -field(#rctx{dbname=Val}, dbname) -> Val; -field(#rctx{username=Val}, username) -> Val; +field(#rctx{type = Val}, type) -> csrt_util:convert_type(Val); +field(#rctx{dbname = Val}, dbname) -> Val; +field(#rctx{username = Val}, username) -> Val; %%field(#rctx{path=Val}, path) -> Val; -field(#rctx{db_open=Val}, db_open) -> Val; -field(#rctx{docs_read=Val}, docs_read) -> Val; -field(#rctx{rows_read=Val}, rows_read) -> Val; -field(#rctx{changes_processed=Val}, changes_processed) -> Val; -field(#rctx{changes_returned=Val}, changes_returned) -> Val; -field(#rctx{ioq_calls=Val}, ioq_calls) -> Val; -field(#rctx{io_bytes_read=Val}, io_bytes_read) -> Val; -field(#rctx{io_bytes_written=Val}, io_bytes_written) -> Val; -field(#rctx{js_evals=Val}, js_evals) -> Val; -field(#rctx{js_filter=Val}, js_filter) -> Val; -field(#rctx{js_filtered_docs=Val}, js_filtered_docs) -> Val; -field(#rctx{mango_eval_match=Val}, mango_eval_match) -> Val; -field(#rctx{get_kv_node=Val}, get_kv_node) -> Val; -field(#rctx{get_kp_node=Val}, get_kp_node) -> Val. +field(#rctx{db_open = Val}, db_open) -> Val; +field(#rctx{docs_read = Val}, docs_read) -> Val; +field(#rctx{rows_read = Val}, rows_read) -> Val; +field(#rctx{changes_processed = Val}, changes_processed) -> Val; +field(#rctx{changes_returned = Val}, changes_returned) -> Val; +field(#rctx{ioq_calls = Val}, ioq_calls) -> Val; +field(#rctx{io_bytes_read = Val}, io_bytes_read) -> Val; +field(#rctx{io_bytes_written = Val}, io_bytes_written) -> Val; +field(#rctx{js_evals = Val}, js_evals) -> Val; +field(#rctx{js_filter = Val}, js_filter) -> Val; +field(#rctx{js_filtered_docs = Val}, js_filtered_docs) -> Val; +field(#rctx{mango_eval_match = Val}, mango_eval_match) -> Val; +field(#rctx{get_kv_node = Val}, get_kv_node) -> Val; +field(#rctx{get_kp_node = Val}, get_kp_node) -> Val. curry_field(Field) -> fun(Ele) -> field(Ele, Field) end. @@ -171,4 +171,3 @@ sorted_by(KeyFun, ValFun, AggFun) -> shortened(sorted(group_by(KeyFun, ValFun, A to_json_list(List) when is_list(List) -> lists:map(fun csrt_util:to_json/1, List). - diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index a3135b97ef8..b085a81fe20 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -39,7 +39,6 @@ -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("couch_stats_resource_tracker.hrl"). - -record(st, {}). %% @@ -58,9 +57,9 @@ create_pid_ref() -> -spec new_context(Type :: rctx_type(), Nonce :: nonce()) -> rctx(). new_context(Type, Nonce) -> #rctx{ - nonce = Nonce, - pid_ref = create_pid_ref(), - type = Type + nonce = Nonce, + pid_ref = create_pid_ref(), + type = Type }. -spec set_context_dbname(DbName, PidRef) -> boolean() when @@ -88,7 +87,7 @@ set_context_username(UserName, PidRef) -> update_element(PidRef, [{#rctx.username, UserName}]). -spec get_context_type(Rctx :: rctx()) -> rctx_type(). -get_context_type(#rctx{type=Type}) -> +get_context_type(#rctx{type = Type}) -> Type. -spec set_context_type(Type, PidRef) -> boolean() when @@ -103,7 +102,7 @@ create_resource(#rctx{} = Rctx) -> -spec destroy_resource(PidRef :: maybe_pid_ref()) -> boolean(). destroy_resource(undefined) -> false; -destroy_resource({_,_}=PidRef) -> +destroy_resource({_, _} = PidRef) -> catch ets:delete(?MODULE, PidRef). -spec get_resource(PidRef :: maybe_pid_ref()) -> maybe_rctx(). @@ -111,7 +110,7 @@ get_resource(undefined) -> undefined; get_resource(PidRef) -> catch case ets:lookup(?MODULE, PidRef) of - [#rctx{}=Rctx] -> + [#rctx{} = Rctx] -> Rctx; [] -> undefined @@ -126,17 +125,17 @@ get_rctx_field(Field) -> maps:get(Field, ?KEYS_TO_FIELDS). -spec update_counter(PidRef, Field, Count) -> non_neg_integer() when - PidRef :: maybe_pid_ref(), - Field :: rctx_field(), - Count :: non_neg_integer(). + PidRef :: maybe_pid_ref(), + Field :: rctx_field(), + Count :: non_neg_integer(). update_counter(undefined, _Field, _Count) -> 0; -update_counter({_Pid,_Ref}=PidRef, Field, Count) when Count >= 0 -> +update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> %% TODO: mem3 crashes without catch, why do we lose the stats table? case is_rctx_field(Field) of true -> Update = {get_rctx_field(Field), Count}, - catch ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref=PidRef}); + catch ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref = PidRef}); false -> 0 end. @@ -146,14 +145,14 @@ inc(PidRef, Field) -> inc(PidRef, Field, 1). -spec inc(PidRef, Field, N) -> non_neg_integer() when - PidRef :: maybe_pid_ref(), - Field :: rctx_field(), - N :: non_neg_integer(). + PidRef :: maybe_pid_ref(), + Field :: rctx_field(), + N :: non_neg_integer(). inc(undefined, _Field, _) -> 0; inc(_PidRef, _Field, 0) -> 0; -inc({_Pid,_Ref}=PidRef, Field, N) when is_integer(N) andalso N >= 0 -> +inc({_Pid, _Ref} = PidRef, Field, N) when is_integer(N) andalso N >= 0 -> case is_rctx_field(Field) of true -> update_counter(PidRef, Field, N); @@ -192,7 +191,6 @@ handle_cast(_Msg, State) -> -spec update_element(PidRef :: maybe_pid_ref(), Updates :: [tuple()]) -> boolean(). update_element(undefined, _Update) -> false; -update_element({_Pid,_Ref}=PidRef, Update) -> +update_element({_Pid, _Ref} = PidRef, Update) -> %% TODO: should we take any action when the update fails? catch ets:update_element(?MODULE, PidRef, Update). - diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index d108ffffc84..e1ece8c88e3 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -58,7 +58,6 @@ field/2 ]). - -include_lib("couch_stats_resource_tracker.hrl"). -spec is_enabled() -> boolean(). @@ -123,12 +122,12 @@ make_dt(A, B, Unit) when is_integer(A) andalso is_integer(B) andalso B > A -> -spec convert_type(T) -> binary() | null when T :: #coordinator{} | #rpc_worker{} | undefined. -convert_type(#coordinator{method=Verb0, path=Path, mod=M0, func=F0}) -> +convert_type(#coordinator{method = Verb0, path = Path, mod = M0, func = F0}) -> M = atom_to_binary(M0), F = atom_to_binary(F0), Verb = atom_to_binary(Verb0), <<"coordinator-{", M/binary, ":", F/binary, "}:", Verb/binary, ":", Path/binary>>; -convert_type(#rpc_worker{mod=M0, func=F0, from=From0}) -> +convert_type(#rpc_worker{mod = M0, func = F0, from = From0}) -> M = atom_to_binary(M0), F = atom_to_binary(F0), From = convert_pidref(From0), @@ -156,7 +155,7 @@ convert_ref(Ref) when is_reference(Ref) -> list_to_binary(ref_to_list(Ref)). -spec to_json(Rctx :: rctx()) -> map(). -to_json(#rctx{}=Rctx) -> +to_json(#rctx{} = Rctx) -> #{ updated_at => tutc(Rctx#rctx.updated_at), started_at => tutc(Rctx#rctx.started_at), @@ -323,7 +322,7 @@ make_delta(PidRef) -> Delta. -spec rctx_delta(TA :: Rctx, TB :: Rctx) -> map(). -rctx_delta(#rctx{}=TA, #rctx{}=TB) -> +rctx_delta(#rctx{} = TA, #rctx{} = TB) -> Delta = #{ docs_read => TB#rctx.docs_read - TA#rctx.docs_read, docs_written => TB#rctx.docs_written - TA#rctx.docs_written, @@ -341,7 +340,7 @@ rctx_delta(#rctx{}=TA, #rctx{}=TB) -> %% TODO: reevaluate this decision %% Only return non zero (and also positive) delta fields %% NOTE: this can result in Delta's of the form #{dt => 1} - maps:filter(fun(_K,V) -> V > 0 end, Delta); + maps:filter(fun(_K, V) -> V > 0 end, Delta); rctx_delta(_, _) -> undefined. @@ -366,7 +365,7 @@ get_pid_ref() -> get(?PID_REF). -spec get_pid_ref(Rctx :: rctx()) -> pid_ref(). -get_pid_ref(#rctx{pid_ref=PidRef}) -> +get_pid_ref(#rctx{pid_ref = PidRef}) -> PidRef; get_pid_ref(R) -> throw({unexpected, R}). @@ -383,7 +382,7 @@ set_fabric_init_p(Func, Enabled) -> %% Expose Persist for use in test cases outside this module -spec set_fabric_init_p(Func, Enabled, Persist) -> ok when - Func :: atom(), Enabled :: boolean(), Persist :: boolean(). + Func :: atom(), Enabled :: boolean(), Persist :: boolean(). set_fabric_init_p(Func, Enabled, Persist) -> Key = fabric_conf_key(Func), ok = config:set_boolean(?CSRT_INIT_P, Key, Enabled, Persist). diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index eaadab70c9c..dca8c40e8db 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -42,7 +42,6 @@ csrt_logger_works_test_() -> ] }. - csrt_logger_matchers_test_() -> { foreach, @@ -65,10 +64,11 @@ make_docs(Count) -> fun(I) -> #doc{ id = ?l2b("foo_" ++ integer_to_list(I)), - body={[{<<"value">>, I}]} + body = {[{<<"value">>, I}]} } end, - lists:seq(1, Count)). + lists:seq(1, Count) + ). setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), @@ -82,18 +82,31 @@ setup() -> Method = 'GET', Path = "/" ++ ?b2l(DbName) ++ "/_all_docs", Nonce = couch_util:to_hex(crypto:strong_rand_bytes(5)), - Req = #httpd{method=Method, nonce=Nonce}, + Req = #httpd{method = Method, nonce = Nonce}, {_, _} = PidRef = csrt:create_coordinator_context(Req, Path), MArgs = #mrargs{include_docs = false}, _Res = fabric:all_docs(DbName, [?ADMIN_CTX], fun view_cb/2, [], MArgs), Rctx = load_rctx(PidRef), - ok = config:set("csrt_logger.matchers_threshold", "docs_read", integer_to_list(?THRESHOLD_DOCS_READ), false), - ok = config:set("csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false), - ok = config:set("csrt_logger.matchers_threshold", "rows_read", integer_to_list(?THRESHOLD_ROWS_READ), false), - ok = config:set("csrt_logger.matchers_threshold", "worker_changes_processed", integer_to_list(?THRESHOLD_CHANGES), false), + ok = config:set( + "csrt_logger.matchers_threshold", "docs_read", integer_to_list(?THRESHOLD_DOCS_READ), false + ), + ok = config:set( + "csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false + ), + ok = config:set( + "csrt_logger.matchers_threshold", "rows_read", integer_to_list(?THRESHOLD_ROWS_READ), false + ), + ok = config:set( + "csrt_logger.matchers_threshold", + "worker_changes_processed", + integer_to_list(?THRESHOLD_CHANGES), + false + ), ok = config:set("csrt_logger.dbnames_io", "foo", integer_to_list(?THRESHOLD_DBNAME_IO), false), ok = config:set("csrt_logger.dbnames_io", "bar", integer_to_list(?THRESHOLD_DBNAME_IO), false), - ok = config:set("csrt_logger.dbnames_io", "foo/bar", integer_to_list(?THRESHOLD_DBNAME_IO), false), + ok = config:set( + "csrt_logger.dbnames_io", "foo/bar", integer_to_list(?THRESHOLD_DBNAME_IO), false + ), csrt_logger:reload_matchers(), #{ctx => Ctx, dbname => DbName, rctx => Rctx, rctxs => rctxs()}. @@ -124,31 +137,37 @@ rctx_gen(Opts0) -> ioq_calls => R, rows_read => R, type => TypeGen, - '_do_changes' => true %% Hack because we need to modify both fields + %% Hack because we need to modify both fields + '_do_changes' => true }, Opts = maps:merge(Base, Opts0), - csrt_util:map_to_rctx(maps:fold( - fun - %% Hack for changes because we need to modify both changes_processed - %% and changes_returned but the latter must be <= the former - ('_do_changes', V, Acc) -> - case V of - true -> - Processed = R(), - Returned = (one_of([0, 0, 1, Processed, rand:uniform(Processed)]))(), - maps:put( - changes_processed, - Processed, - maps:put(changes_returned, Returned, Acc)); - _ -> - Acc - end; - (K, F, Acc) when is_function(F) -> - maps:put(K, F(), Acc); - (K, V, Acc) -> - maps:put(K, V, Acc) - end, #{}, Opts - )). + csrt_util:map_to_rctx( + maps:fold( + fun + %% Hack for changes because we need to modify both changes_processed + %% and changes_returned but the latter must be <= the former + ('_do_changes', V, Acc) -> + case V of + true -> + Processed = R(), + Returned = (one_of([0, 0, 1, Processed, rand:uniform(Processed)]))(), + maps:put( + changes_processed, + Processed, + maps:put(changes_returned, Returned, Acc) + ); + _ -> + Acc + end; + (K, F, Acc) when is_function(F) -> + maps:put(K, F(), Acc); + (K, V, Acc) -> + maps:put(K, V, Acc) + end, + #{}, + Opts + ) + ). rctxs() -> [rctx_gen() || _ <- lists:seq(1, ?RCTX_COUNT)]. @@ -304,7 +323,8 @@ t_matcher_on_dbnames_io(#{rctxs := Rctxs0}) -> ). load_rctx(PidRef) -> - timer:sleep(50), %% Add slight delay to accumulate RPC response deltas + %% Add slight delay to accumulate RPC response deltas + timer:sleep(50), csrt:get_resource(PidRef). view_cb({row, Row}, Acc) -> diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index e26dfc7cc32..385e4d4ce8a 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -22,7 +22,6 @@ -define(DEBUG_ENABLED, false). - csrt_context_test_() -> { setup, @@ -68,7 +67,7 @@ csrt_fabric_test_() -> { "CSRT fabric tests with a DDoc present", foreach, - fun() -> setup_ddoc(<<"_design/foo">>, <<"bar">>) end, + fun() -> setup_ddoc(<<"_design/foo">>, <<"bar">>) end, fun teardown/1, ddoc_test_funs() }. @@ -78,10 +77,11 @@ make_docs(Count) -> fun(I) -> #doc{ id = ?l2b("foo_" ++ integer_to_list(I)), - body={[{<<"value">>, I}]} + body = {[{<<"value">>, I}]} } end, - lists:seq(1, Count)). + lists:seq(1, Count) + ). setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), @@ -107,19 +107,23 @@ setup_ddoc(DDocId, ViewName) -> {<<"language">>, <<"javascript">>}, { <<"views">>, - {[{ - ViewName, - {[ - {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} - ]} - }]} + {[ + { + ViewName, + {[ + {<<"map">>, <<"function(doc) { emit(doc.value, null); }">>} + ]} + } + ]} }, { <<"filters">>, - {[{ - <<"even">>, - <<"function(doc) { return (doc.value % 2 == 0); }">> - }]} + {[ + { + <<"even">>, + <<"function(doc) { return (doc.value % 2 == 0); }">> + } + ]} } ]} ), @@ -245,7 +249,6 @@ t_get_doc({_Ctx, DbName, _View}) -> ok = nonzero_local_io_assert(Rctx, io_sum), ok = assert_teardown(PidRef). - t_put_doc({_Ctx, DbName, View}) -> pdebug(dbname, DbName), DocId = "bar_put_1919", @@ -320,7 +323,6 @@ t_changes({_Ctx, DbName, View}) -> ok = nonzero_local_io_assert(Rctx), ok = assert_teardown(PidRef). - t_changes_limit_zero({_Ctx, DbName, _View}) -> Context = #{ method => 'GET', @@ -329,7 +331,7 @@ t_changes_limit_zero({_Ctx, DbName, _View}) -> {PidRef, Nonce} = coordinator_context(Context), Rctx0 = load_rctx(PidRef), ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), - _Res = fabric:changes(DbName, fun changes_cb/2, [], #changes_args{limit=0}), + _Res = fabric:changes(DbName, fun changes_cb/2, [], #changes_args{limit = 0}), Rctx = load_rctx(PidRef), ok = rctx_assert(Rctx, #{ nonce => Nonce, @@ -347,7 +349,7 @@ t_changes_limit_zero({_Ctx, DbName, _View}) -> t_changes_filtered({_Ctx, _DbName, _View}) -> false. -t_changes_js_filtered({_Ctx, DbName, {DDocId, _ViewName}=View}) -> +t_changes_js_filtered({_Ctx, DbName, {DDocId, _ViewName} = View}) -> pdebug(dbname, DbName), Method = 'GET', Path = "/" ++ ?b2l(DbName) ++ "/_changes", @@ -452,7 +454,7 @@ pdebug(rctx, Rctx) -> pdbg(Str, Args) -> ?DEBUG_ENABLED andalso ?debugFmt(Str, Args). -convert_pidref({_, _}=PidRef) -> +convert_pidref({_, _} = PidRef) -> csrt_util:convert_pidref(PidRef); convert_pidref(PidRef) when is_binary(PidRef) -> PidRef; @@ -466,12 +468,12 @@ rctx_assert(Rctx, Asserts0) -> js_filtered_docs => 0, write_kp_node => 0, write_kv_node => 0, - nonce => undefined, - db_open => 0, - rows_read => 0, - docs_read => 0, - docs_written => 0, - pid_ref => undefined + nonce => undefined, + db_open => 0, + rows_read => 0, + docs_read => 0, + docs_written => 0, + pid_ref => undefined }, Asserts = maps:merge( DefaultAsserts, @@ -528,7 +530,7 @@ ddoc_dependent_local_io_assert(Rctx, {_DDoc, _ViewName}) -> coordinator_context(#{method := Method, path := Path}) -> Nonce = couch_util:to_hex(crypto:strong_rand_bytes(5)), - Req = #httpd{method=Method, nonce=Nonce}, + Req = #httpd{method = Method, nonce = Nonce}, {_, _} = PidRef = csrt:create_coordinator_context(Req, Path), {PidRef, Nonce}. @@ -548,7 +550,7 @@ assert_gt() -> assert_gt(0). assert_gt(N) -> - fun(K, RV) -> ?assert(RV > N, {K, RV, N}) end. + fun(K, RV) -> ?assert(RV > N, {K, RV, N}) end. assert_gte(N) -> fun(K, RV) -> ?assert(RV >= N, {K, RV, N}) end. @@ -568,5 +570,6 @@ configure_filter(DbName, DDocId, Req, FName) -> {fetch, custom, Style, Req, DIR, FName}. load_rctx(PidRef) -> - timer:sleep(50), %% Add slight delay to accumulate RPC response deltas + %% Add slight delay to accumulate RPC response deltas + timer:sleep(50), csrt_util:to_json(csrt:get_resource(PidRef)). diff --git a/src/fabric/test/eunit/fabric_rpc_tests.erl b/src/fabric/test/eunit/fabric_rpc_tests.erl index f5e4e52f691..6391ffbb774 100644 --- a/src/fabric/test/eunit/fabric_rpc_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_tests.erl @@ -103,13 +103,16 @@ t_no_config_db_create_fails_for_shard_rpc(DbName) -> end, case csrt:is_enabled() of true -> - ?assertMatch( %% allow for {Ref, {rexi_EXIT, error}, {delta, D}} + %% allow for {Ref, {rexi_EXIT, error}, {delta, D}} + ?assertMatch( {Ref, {'rexi_EXIT', {{error, missing_target}, _}}, _}, - Resp); + Resp + ); false -> ?assertMatch( {Ref, {'rexi_EXIT', {{error, missing_target}, _}}}, - Resp) + Resp + ) end. t_db_create_with_config(DbName) -> From 020743fe6793b5abb5bbefb14fdfd63ccc45afda Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 9 Apr 2025 15:25:49 -0700 Subject: [PATCH 09/54] CI Bump.. --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index ece0ca891d1..45c5eb8adae 100644 --- a/README.rst +++ b/README.rst @@ -1,6 +1,7 @@ Apache CouchDB README ===================== + +---------+ | |1| |2| | +---------+ From 975818e6483965447864a0a60daa00c8d89f11d4 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 9 Apr 2025 16:43:55 -0700 Subject: [PATCH 10/54] Create delta prior to deleting the context --- src/rexi/src/rexi_server.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index 8ba1ee2e58c..038612f5a5d 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -147,6 +147,7 @@ init_p(From, {M, F, A}, Nonce) -> csrt:destroy_context(), ok; Class:Reason:Stack0 -> + Delta = csrt:make_delta(), csrt:destroy_context(), Stack = clean_stack(Stack0), {ClientPid, _ClientRef} = From, @@ -163,7 +164,7 @@ init_p(From, {M, F, A}, Nonce) -> ] ), exit(#error{ - delta = csrt:make_delta(), + delta = Delta, timestamp = os:timestamp(), reason = {Class, Reason}, mfa = {M, F, A}, From a8dd0d62145715fe35f75d75665df64f0c03fb5c Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Thu, 24 Apr 2025 17:30:34 -0700 Subject: [PATCH 11/54] Updates based on PR feedback --- .../src/couch_stats_resource_tracker.hrl | 2 ++ src/couch_stats/src/csrt.erl | 2 +- src/couch_stats/src/csrt_logger.erl | 2 +- src/couch_stats/src/csrt_server.erl | 23 +++++++++++++------ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 82a7ead7ba9..01589102c72 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -71,6 +71,8 @@ ?COUCH_BT_WRITE_KV_NODE => #rctx.?COUCH_BT_WRITE_KV_NODE }). +-type throw(_Reason) :: no_return(). + -type pid_ref() :: {pid(), reference()}. -type maybe_pid_ref() :: pid_ref() | undefined. -type maybe_pid() :: pid() | undefined. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 2d8dc8670af..4faac5172b7 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -129,7 +129,7 @@ destroy_pid_ref(_PidRef) -> %% csrt_server:create_resource(Rctx). -spec create_worker_context(From, MFA, Nonce) -> pid_ref() | false when - From :: pid_ref(), MFA :: mfa(), Nonce :: term(). + From :: pid_ref(), MFA :: mfa(), Nonce :: nonce(). create_worker_context(From, {M, F, _A}, Nonce) -> case is_enabled() of true -> diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index b31e18afd3d..a4b28043e29 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -234,7 +234,7 @@ handle_cast(_Msg, State) -> {noreply, State, 0}. handle_info(restart_config_listener, State) -> - ok = config:listen_for_changes(?MODULE, nil), + ok = subscribe_changes(), {noreply, State}; handle_info(_Msg, St) -> {noreply, St}. diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index b085a81fe20..4b471132041 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -95,32 +95,36 @@ get_context_type(#rctx{type = Type}) -> set_context_type(Type, PidRef) -> update_element(PidRef, [{#rctx.type, Type}]). --spec create_resource(Rctx :: rctx()) -> true. +-spec create_resource(Rctx :: rctx()) -> boolean(). create_resource(#rctx{} = Rctx) -> - catch ets:insert(?MODULE, Rctx). + (catch ets:insert(?MODULE, Rctx)) == true. -spec destroy_resource(PidRef :: maybe_pid_ref()) -> boolean(). destroy_resource(undefined) -> false; destroy_resource({_, _} = PidRef) -> - catch ets:delete(?MODULE, PidRef). + (catch ets:delete(?MODULE, PidRef)) == true. -spec get_resource(PidRef :: maybe_pid_ref()) -> maybe_rctx(). get_resource(undefined) -> undefined; get_resource(PidRef) -> - catch case ets:lookup(?MODULE, PidRef) of + try ets:lookup(?MODULE, PidRef) of [#rctx{} = Rctx] -> Rctx; [] -> undefined + catch + _:_ -> + undefined end. -spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). is_rctx_field(Field) -> maps:is_key(Field, ?KEYS_TO_FIELDS). --spec get_rctx_field(Field :: rctx_field()) -> non_neg_integer(). +-spec get_rctx_field(Field :: rctx_field()) -> non_neg_integer() + | throw({badkey, Key :: any()}). get_rctx_field(Field) -> maps:get(Field, ?KEYS_TO_FIELDS). @@ -135,7 +139,12 @@ update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> case is_rctx_field(Field) of true -> Update = {get_rctx_field(Field), Count}, - catch ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref = PidRef}); + try + ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref = PidRef}) + catch + _:_ -> + 0 + end; false -> 0 end. @@ -152,7 +161,7 @@ inc(undefined, _Field, _) -> 0; inc(_PidRef, _Field, 0) -> 0; -inc({_Pid, _Ref} = PidRef, Field, N) when is_integer(N) andalso N >= 0 -> +inc({_Pid, _Ref} = PidRef, Field, N) when is_integer(N) andalso N > 0 -> case is_rctx_field(Field) of true -> update_counter(PidRef, Field, N); From f280d1b1256b2e791dd9295a048bbaa0f2e74646 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 7 May 2025 13:43:57 -0700 Subject: [PATCH 12/54] Address more PR feedback --- .../src/couch_stats_resource_tracker.hrl | 1 + src/couch_stats/src/csrt.erl | 2 +- src/couch_stats/src/csrt_query.erl | 20 +++++------ src/couch_stats/src/csrt_server.erl | 19 +++++++---- src/couch_stats/src/csrt_util.erl | 10 +++--- .../test/eunit/csrt_logger_tests.erl | 34 ++++++++++--------- 6 files changed, 46 insertions(+), 40 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 01589102c72..c37c910ba6e 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -12,6 +12,7 @@ -define(CSRT, "csrt"). -define(CSRT_INIT_P, "csrt.init_p"). +-define(CSRT_ETS, csrt_server). %% CSRT pdict markers -define(DELTA_TA, csrt_delta_ta). diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 4faac5172b7..ffea2c81784 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -235,7 +235,7 @@ destroy_context() -> -spec destroy_context(PidRef :: maybe_pid_ref()) -> ok. destroy_context(undefined) -> ok; -destroy_context({_, _} = PidRef) -> +destroy_context(PidRef) -> csrt_logger:stop_tracker(), destroy_pid_ref(PidRef), ok. diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl index 3de8f417bb1..1708ebebc34 100644 --- a/src/couch_stats/src/csrt_query.erl +++ b/src/couch_stats/src/csrt_query.erl @@ -67,27 +67,23 @@ active_int(all) -> select_by_type(all). select_by_type(coordinators) -> - ets:select(?MODULE, ets:fun2ms(fun(#rctx{type = #coordinator{}} = R) -> R end)); + ets:select(?CSRT_ETS, ets:fun2ms(fun(#rctx{type = #coordinator{}} = R) -> R end)); select_by_type(workers) -> - ets:select(?MODULE, ets:fun2ms(fun(#rctx{type = #rpc_worker{}} = R) -> R end)); + ets:select(?CSRT_ETS, ets:fun2ms(fun(#rctx{type = #rpc_worker{}} = R) -> R end)); select_by_type(all) -> - ets:tab2list(?MODULE). + ets:tab2list(?CSRT_ETS). find_by_nonce(Nonce) -> - %%ets:match_object(?MODULE, ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end)). - [R || R <- ets:match_object(?MODULE, #rctx{nonce = Nonce})]. + csrt_server:match_resource(#rctx{nonce = Nonce}). find_by_pid(Pid) -> - %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{pid_ref={Pid, '_'}, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{pid_ref = {Pid, '_'}})]. + csrt_server:match_resource(#rctx{pid_ref = {Pid, '_'}}). find_by_pidref(PidRef) -> - %%[R || R <- ets:match_object(?MODULE, #rctx{pid_ref=PidRef, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{pid_ref = PidRef})]. + csrt_server:match_resource(#rctx{pid_ref = PidRef}). find_workers_by_pidref(PidRef) -> - %%[R || #rctx{} = R <- ets:match_object(?MODULE, #rctx{type=#rpc_worker{from=PidRef}, _ = '_'})]. - [R || R <- ets:match_object(?MODULE, #rctx{type = #rpc_worker{from = PidRef}})]. + csrt_server:match_resource(#rctx{type = #rpc_worker{from = PidRef}}). field(#rctx{pid_ref = Val}, pid_ref) -> Val; %% NOTE: Pros and cons to doing these convert functions here @@ -154,7 +150,7 @@ group_by(KeyFun, ValFun, AggFun) -> Acc end end, - ets:foldl(FoldFun, #{}, ?MODULE). + ets:foldl(FoldFun, #{}, ?CSRT_ETS). %% Sorts largest first sorted(Map) when is_map(Map) -> diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index 4b471132041..a90e7d87a48 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -29,6 +29,7 @@ get_context_type/1, inc/2, inc/3, + match_resource/1, new_context/2, set_context_dbname/2, set_context_username/2, @@ -97,19 +98,19 @@ set_context_type(Type, PidRef) -> -spec create_resource(Rctx :: rctx()) -> boolean(). create_resource(#rctx{} = Rctx) -> - (catch ets:insert(?MODULE, Rctx)) == true. + (catch ets:insert(?CSRT_ETS, Rctx)) == true. -spec destroy_resource(PidRef :: maybe_pid_ref()) -> boolean(). destroy_resource(undefined) -> false; destroy_resource({_, _} = PidRef) -> - (catch ets:delete(?MODULE, PidRef)) == true. + (catch ets:delete(?CSRT_ETS, PidRef)) == true. -spec get_resource(PidRef :: maybe_pid_ref()) -> maybe_rctx(). get_resource(undefined) -> undefined; get_resource(PidRef) -> - try ets:lookup(?MODULE, PidRef) of + try ets:lookup(?CSRT_ETS, PidRef) of [#rctx{} = Rctx] -> Rctx; [] -> @@ -119,6 +120,12 @@ get_resource(PidRef) -> undefined end. +-spec match_resource(Rctx :: maybe_rctx()) -> [] | [rctx()]. +match_resource(undefined) -> + []; +match_resource(#rctx{} = Rctx) -> + ets:match_object(?CSRT_ETS, Rctx). + -spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). is_rctx_field(Field) -> maps:is_key(Field, ?KEYS_TO_FIELDS). @@ -140,7 +147,7 @@ update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> true -> Update = {get_rctx_field(Field), Count}, try - ets:update_counter(?MODULE, PidRef, Update, #rctx{pid_ref = PidRef}) + ets:update_counter(?CSRT_ETS, PidRef, Update, #rctx{pid_ref = PidRef}) catch _:_ -> 0 @@ -177,7 +184,7 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> - ets:new(?MODULE, [ + ets:new(?CSRT_ETS, [ named_table, public, {decentralized_counters, true}, @@ -202,4 +209,4 @@ update_element(undefined, _Update) -> false; update_element({_Pid, _Ref} = PidRef, Update) -> %% TODO: should we take any action when the update fails? - catch ets:update_element(?MODULE, PidRef, Update). + catch ets:update_element(?CSRT_ETS, PidRef, Update). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index e1ece8c88e3..6e142bc4d24 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -179,7 +179,7 @@ to_json(#rctx{} = Rctx) -> ioq_calls => Rctx#rctx.ioq_calls }. -%% NOTE: this does not do the inverse of to_json, should it conver types? +%% NOTE: this does not do the inverse of to_json, should it convert types? -spec map_to_rctx(Map :: map()) -> rctx(). map_to_rctx(Map) -> maps:fold(fun map_to_rctx_field/3, #rctx{}, Map). @@ -403,8 +403,8 @@ couch_stats_resource_tracker_test_() -> fun teardown/1, [ ?TDEF_FE(t_should_track_init_p), - ?TDEF_FE(t_should_track_init_p_empty), - ?TDEF_FE(t_should_track_init_p_disabled), + ?TDEF_FE(t_should_not_track_init_p_empty), + ?TDEF_FE(t_should_not_track_init_p_disabled), ?TDEF_FE(t_should_not_track_init_p) ] }. @@ -419,11 +419,11 @@ t_should_track_init_p(_) -> enable_init_p(), [?assert(should_track_init_p(M, F), {M, F}) || [M, F] <- base_metrics()]. -t_should_track_init_p_empty(_) -> +t_should_not_track_init_p_empty(_) -> config:set(?CSRT_INIT_P, "enabled", "true", false), [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. -t_should_track_init_p_disabled(_) -> +t_should_not_track_init_p_disabled(_) -> config:set(?CSRT_INIT_P, "enabled", "false", false), [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index dca8c40e8db..d13c67ee73a 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -15,13 +15,14 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). +-include("../../src/couch_stats_resource_tracker.hrl"). -define(RCTX_RANGE, 1000). -define(RCTX_COUNT, 10000). %% Dirty hack for hidden records as .hrl is only in src/ --define(RCTX_RPC, {rpc_worker, foo, bar, {self(), make_ref()}}). --define(RCTX_COORDINATOR, {coordinator, foo, bar, 'GET', "/foo/_all_docs"}). +-define(RCTX_RPC, #rpc_worker{from = {self(), make_ref()}}). +-define(RCTX_COORDINATOR, #coordinator{method = 'GET', path = "/foo/_all_docs"}). -define(THRESHOLD_DBNAME, <<"foo">>). -define(THRESHOLD_DBNAME_IO, 91). @@ -33,8 +34,8 @@ csrt_logger_works_test_() -> { foreach, - fun setup/0, - fun teardown/1, + fun setup_reporting/0, + fun teardown_reporting/1, [ ?TDEF_FE(t_do_report), ?TDEF_FE(t_do_lifetime_report), @@ -115,6 +116,16 @@ teardown(#{ctx := Ctx, dbname := DbName}) -> ok = meck:unload(ioq), test_util:stop_couch(Ctx). +setup_reporting() -> + Ctx = setup(), + ok = meck:new(couch_log), + ok = meck:expect(couch_log, report, fun(_, _) -> true end), + Ctx. + +teardown_reporting(Ctx) -> + ok = meck:unload(couch_log), + teardown(Ctx). + rctx_gen() -> rctx_gen(#{}). @@ -175,22 +186,17 @@ rctxs() -> t_do_report(#{rctx := Rctx}) -> JRctx = csrt_util:to_json(Rctx), ReportName = "foo", - ok = meck:new(couch_log), - ok = meck:expect(couch_log, report, fun(_, _) -> true end), ?assert(csrt_logger:do_report(ReportName, Rctx), "CSRT _logger:do_report " ++ ReportName), ?assert(meck:validate(couch_log), "CSRT do_report"), ?assert(meck:validate(couch_log), "CSRT validate couch_log"), ?assert( meck:called(couch_log, report, [ReportName, JRctx]), "CSRT couch_log:report" - ), - ok = meck:unload(couch_log). + ). t_do_lifetime_report(#{rctx := Rctx}) -> JRctx = csrt_util:to_json(Rctx), ReportName = "csrt-pid-usage-lifetime", - ok = meck:new(couch_log), - ok = meck:expect(couch_log, report, fun(_, _) -> true end), ?assert( csrt_logger:do_lifetime_report(Rctx), "CSRT _logger:do_report " ++ ReportName @@ -199,21 +205,17 @@ t_do_lifetime_report(#{rctx := Rctx}) -> ?assert( meck:called(couch_log, report, [ReportName, JRctx]), "CSRT couch_log:report" - ), - ok = meck:unload(couch_log). + ). t_do_status_report(#{rctx := Rctx}) -> JRctx = csrt_util:to_json(Rctx), ReportName = "csrt-pid-usage-status", - ok = meck:new(couch_log), - ok = meck:expect(couch_log, report, fun(_, _) -> true end), ?assert(csrt_logger:do_status_report(Rctx), "csrt_logger:do_ " ++ ReportName), ?assert(meck:validate(couch_log), "CSRT validate couch_log"), ?assert( meck:called(couch_log, report, [ReportName, JRctx]), "CSRT couch_log:report" - ), - ok = meck:unload(couch_log). + ). t_matcher_on_dbname(#{rctx := _Rctx, rctxs := Rctxs0}) -> %% Make sure we have at least one match From ae419d67de4e4bc0cb3c7788d9e4271c44a8d293 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 7 May 2025 15:16:20 -0700 Subject: [PATCH 13/54] Fix erlfmt-check --- src/couch_stats/src/csrt_server.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index a90e7d87a48..fa22558ba38 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -130,7 +130,8 @@ match_resource(#rctx{} = Rctx) -> is_rctx_field(Field) -> maps:is_key(Field, ?KEYS_TO_FIELDS). --spec get_rctx_field(Field :: rctx_field()) -> non_neg_integer() +-spec get_rctx_field(Field :: rctx_field()) -> + non_neg_integer() | throw({badkey, Key :: any()}). get_rctx_field(Field) -> maps:get(Field, ?KEYS_TO_FIELDS). From 57c19cd5e17d6079101a980c84bcd8787b8b267b Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 7 May 2025 16:59:29 -0700 Subject: [PATCH 14/54] Rework and fix csrt_util init_p ini lookup tests --- src/couch_stats/src/csrt_util.erl | 32 ++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 6e142bc4d24..890f653f323 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -70,7 +70,7 @@ is_enabled_init_p() -> -spec should_track_init_p(Mod :: atom(), Func :: atom()) -> boolean(). should_track_init_p(fabric_rpc, Func) -> - config:get_boolean(?CSRT_INIT_P, fabric_conf_key(Func), false); + is_enabled_init_p() andalso config:get_boolean(?CSRT_INIT_P, fabric_conf_key(Func), false); should_track_init_p(_Mod, _Func) -> false. @@ -404,6 +404,7 @@ couch_stats_resource_tracker_test_() -> [ ?TDEF_FE(t_should_track_init_p), ?TDEF_FE(t_should_not_track_init_p_empty), + ?TDEF_FE(t_should_not_track_init_p_empty_and_disabled), ?TDEF_FE(t_should_not_track_init_p_disabled), ?TDEF_FE(t_should_not_track_init_p) ] @@ -420,11 +421,17 @@ t_should_track_init_p(_) -> [?assert(should_track_init_p(M, F), {M, F}) || [M, F] <- base_metrics()]. t_should_not_track_init_p_empty(_) -> - config:set(?CSRT_INIT_P, "enabled", "true", false), + disable_init_p_metrics(), + enable_init_p([]), + [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. + +t_should_not_track_init_p_empty_and_disabled(_) -> + disable_init_p(), [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. t_should_not_track_init_p_disabled(_) -> - config:set(?CSRT_INIT_P, "enabled", "false", false), + enable_init_p_metrics(), + disable_init_p(), [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- base_metrics()]. t_should_not_track_init_p(_) -> @@ -442,8 +449,27 @@ enable_init_p() -> enable_init_p(Metrics) -> config:set(?CSRT_INIT_P, "enabled", "true", false), + enable_init_p_metrics(Metrics). + +enable_init_p_metrics() -> + enable_init_p(base_metrics()). + +enable_init_p_metrics(Metrics) -> [set_fabric_init_p(F, true, false) || [_, F] <- Metrics]. +disable_init_p() -> + disable_init_p(base_metrics()). + +disable_init_p(Metrics) -> + config:set(?CSRT_INIT_P, "enabled", "false", false), + disable_init_p_metrics(Metrics). + +disable_init_p_metrics() -> + disable_init_p_metrics(base_metrics()). + +disable_init_p_metrics(Metrics) -> + [set_fabric_init_p(F, false, false) || [_, F] <- Metrics]. + base_metrics() -> [ [fabric_rpc, all_docs], From 2bfefd48e1a2c22f2a925fbcaaed8f19260fa276 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Thu, 8 May 2025 16:13:49 -0700 Subject: [PATCH 15/54] Rework delta handling back to normal process_message semantics --- src/couch_stats/src/csrt.erl | 4 +- src/couch_stats/src/csrt_util.erl | 47 ++++++++++++------- .../test/eunit/csrt_logger_tests.erl | 1 + .../test/eunit/csrt_server_tests.erl | 2 + src/fabric/src/fabric_util.erl | 45 ++++++++---------- .../test/eunit/fabric_rpc_purge_tests.erl | 4 +- src/fabric/test/eunit/fabric_rpc_tests.erl | 2 +- src/mem3/src/mem3_rpc.erl | 41 +++++++--------- src/rexi/src/rexi.erl | 6 ++- src/rexi/src/rexi_server.erl | 3 +- src/rexi/src/rexi_utils.erl | 36 ++++++++------ src/rexi/test/rexi_tests.erl | 20 ++++---- 12 files changed, 113 insertions(+), 98 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index ffea2c81784..534696d93af 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -448,7 +448,9 @@ couch_stats_resource_tracker_test_() -> }. setup() -> - test_util:start_couch(). + Ctx = test_util:start_couch(), + config:set_boolean(?CSRT, "randomize_testing", false, false), + Ctx. teardown(Ctx) -> test_util:stop_couch(Ctx). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 890f653f323..8c773a44f1e 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -60,9 +60,22 @@ -include_lib("couch_stats_resource_tracker.hrl"). +-ifdef(TEST). +-spec is_enabled() -> boolean(). +is_enabled() -> + %% randomly enable CSRT during testing to handle unexpected failures + case config:get_boolean(?CSRT, "randomize_testing", true) of + true -> + rand:uniform(100) > 80; + false -> + config:get_boolean(?CSRT, "enabled", true) + end. +-else. -spec is_enabled() -> boolean(). is_enabled() -> + %% TODO: toggle back to false before merging config:get_boolean(?CSRT, "enabled", true). +-endif. -spec is_enabled_init_p() -> boolean(). is_enabled_init_p() -> @@ -264,23 +277,17 @@ field(changes_processed, #rctx{changes_processed = Val}) -> field(ioq_calls, #rctx{ioq_calls = Val}) -> Val. -add_delta({A}, Delta) -> {A, Delta}; -add_delta({A, B}, Delta) -> {A, B, Delta}; -add_delta({A, B, C}, Delta) -> {A, B, C, Delta}; -add_delta({A, B, C, D}, Delta) -> {A, B, C, D, Delta}; -add_delta({A, B, C, D, E}, Delta) -> {A, B, C, D, E, Delta}; -add_delta({A, B, C, D, E, F}, Delta) -> {A, B, C, D, E, F, Delta}; -add_delta({A, B, C, D, E, F, G}, Delta) -> {A, B, C, D, E, F, G, Delta}; -add_delta(T, _Delta) -> T. - -extract_delta({A, {delta, Delta}}) -> {{A}, Delta}; -extract_delta({A, B, {delta, Delta}}) -> {{A, B}, Delta}; -extract_delta({A, B, C, {delta, Delta}}) -> {{A, B, C}, Delta}; -extract_delta({A, B, C, D, {delta, Delta}}) -> {{A, B, C, D}, Delta}; -extract_delta({A, B, C, D, E, {delta, Delta}}) -> {{A, B, C, D, E}, Delta}; -extract_delta({A, B, C, D, E, F, {delta, Delta}}) -> {{A, B, C, D, E, F}, Delta}; -extract_delta({A, B, C, D, E, F, G, {delta, Delta}}) -> {{A, B, C, D, E, F, G}, Delta}; -extract_delta(T) -> {T, undefined}. +add_delta(T, {delta, undefined}) -> + T; +add_delta(T, {delta, _} = Delta) -> + {T, Delta}; +add_delta(T, _Delta) -> + T. + +extract_delta({Msg, {delta, Delta}}) -> + {Msg, Delta}; +extract_delta(Msg) -> + {Msg, undefined}. -spec get_delta(PidRef :: maybe_pid_ref()) -> tagged_delta(). get_delta(PidRef) -> @@ -306,6 +313,8 @@ maybe_add_delta(T, Delta) -> maybe_add_delta_int(T, undefined) -> T; +maybe_add_delta_int(T, {delta, undefined}) -> + T; maybe_add_delta_int(T, Delta) when is_map(Delta) -> maybe_add_delta_int(T, {delta, Delta}); maybe_add_delta_int(T, {delta, _} = Delta) -> @@ -411,7 +420,9 @@ couch_stats_resource_tracker_test_() -> }. setup() -> - test_util:start_couch(). + Ctx = test_util:start_couch(), + config:set_boolean(?CSRT, "randomize_testing", false, false), + Ctx. teardown(Ctx) -> test_util:stop_couch(Ctx). diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index d13c67ee73a..7bb3bcbeb0e 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -73,6 +73,7 @@ make_docs(Count) -> setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), + config:set_boolean(?CSRT, "randomize_testing", false, false), ok = meck:new(ioq, [passthrough]), ok = meck:expect(ioq, bypass, fun(_, _) -> false end), DbName = ?tempdb(), diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index 385e4d4ce8a..fa39c82e561 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -15,6 +15,7 @@ -include_lib("couch/include/couch_db.hrl"). -include_lib("couch/include/couch_eunit.hrl"). -include_lib("couch_mrview/include/couch_mrview.hrl"). +-include("../../src/couch_stats_resource_tracker.hrl"). -define(DOCS_COUNT, 100). -define(DDOCS_COUNT, 1). @@ -85,6 +86,7 @@ make_docs(Count) -> setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), + config:set_boolean(?CSRT, "randomize_testing", false, false), ok = meck:new(ioq, [passthrough]), ok = meck:expect(ioq, bypass, fun(_, _) -> false end), DbName = ?tempdb(), diff --git a/src/fabric/src/fabric_util.erl b/src/fabric/src/fabric_util.erl index f93644d2e93..93de4b356b3 100644 --- a/src/fabric/src/fabric_util.erl +++ b/src/fabric/src/fabric_util.erl @@ -136,36 +136,29 @@ get_shard([#shard{node = Node, name = Name} | Rest], Opts, Timeout, Factor) -> MFA = {fabric_rpc, open_shard, [Name, [{timeout, Timeout} | Opts]]}, Ref = rexi:cast(Node, self(), MFA, [sync]), try - await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) + receive + {Ref, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), + case Msg of + {ok, Db} -> + {ok, Db}; + {'rexi_EXIT', {{unauthorized, _} = Error, _}} -> + throw(Error); + {'rexi_EXIT', {{forbidden, _} = Error, _}} -> + throw(Error); + Reason -> + couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), + get_shard(Rest, Opts, Timeout, Factor) + end + after Timeout -> + couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), + get_shard(Rest, Opts, Factor * Timeout, Factor) + end after rexi_monitor:stop(Mon) end. -await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) -> - receive - Msg0 -> - {Msg, Delta} = csrt:extract_delta(Msg0), - csrt:accumulate_delta(Delta), - case Msg of - {Ref, {ok, Db}} -> - {ok, Db}; - {Ref, {'rexi_EXIT', {{unauthorized, _} = Error, _}}} -> - throw(Error); - {Ref, {'rexi_EXIT', {{forbidden, _} = Error, _}}} -> - throw(Error); - {Ref, Reason} -> - couch_log:debug("Failed to open shard ~p because: ~p", [Name, Reason]), - get_shard(Rest, Opts, Timeout, Factor); - %% {OldRef, {ok, Db}} -> <-- stale db resp that got here late, should we do something? - _ -> - %% Got a message from an old Ref that timed out, try again - await_shard_response(Ref, Name, Rest, Opts, Factor, Timeout) - end - after Timeout -> - couch_log:debug("Failed to open shard ~p after: ~p", [Name, Timeout]), - get_shard(Rest, Opts, Factor * Timeout, Factor) - end. - get_db_timeout(N, Factor, MinTimeout, infinity) -> % MaxTimeout may be infinity so we just use the largest Erlang small int to % avoid blowing up the arithmetic diff --git a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl index c7a36fbe342..57c533ccd6c 100644 --- a/src/fabric/test/eunit/fabric_rpc_purge_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_purge_tests.erl @@ -262,9 +262,9 @@ rpc_update_doc(DbName, Doc, Opts) -> fabric_rpc:update_docs(DbName, [Doc], Opts), Reply = test_util:wait(fun() -> receive - {Ref, Reply} -> + {Ref, {Reply, {delta, _}}} -> Reply; - {Ref, Reply, {delta, _}} -> + {Ref, Reply} -> Reply after 0 -> wait diff --git a/src/fabric/test/eunit/fabric_rpc_tests.erl b/src/fabric/test/eunit/fabric_rpc_tests.erl index 6391ffbb774..03596290a5e 100644 --- a/src/fabric/test/eunit/fabric_rpc_tests.erl +++ b/src/fabric/test/eunit/fabric_rpc_tests.erl @@ -105,7 +105,7 @@ t_no_config_db_create_fails_for_shard_rpc(DbName) -> true -> %% allow for {Ref, {rexi_EXIT, error}, {delta, D}} ?assertMatch( - {Ref, {'rexi_EXIT', {{error, missing_target}, _}}, _}, + {Ref, {{'rexi_EXIT', {{error, missing_target}, _}}, _}}, Resp ); false -> diff --git a/src/mem3/src/mem3_rpc.erl b/src/mem3/src/mem3_rpc.erl index 0711cfb67dc..47de77490f9 100644 --- a/src/mem3/src/mem3_rpc.erl +++ b/src/mem3/src/mem3_rpc.erl @@ -378,34 +378,27 @@ rexi_call(Node, MFA, Timeout) -> Mon = rexi_monitor:start([rexi_utils:server_pid(Node)]), Ref = rexi:cast(Node, self(), MFA, [sync]), try - wait_message(Node, Ref, Mon, Timeout) + receive + {Ref, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), + case Msg of + {ok, Reply} -> + Reply; + Error -> + erlang:error(Error) + end; + {rexi_DOWN, Mon, _, Reason0} -> + {Reason, Delta} = csrt:extract_delta(Reason0), + csrt:accumulate_delta(Delta), + erlang:error({rexi_DOWN, {Node, Reason}}) + after Timeout -> + erlang:error(timeout) + end after rexi_monitor:stop(Mon) end. -wait_message(Node, Ref, Mon, Timeout) -> - receive - Msg -> - process_raw_message(Msg, Node, Ref, Mon, Timeout) - after Timeout -> - erlang:error(timeout) - end. - -process_raw_message(Msg0, Node, Ref, Mon, Timeout) -> - {Msg, Delta} = csrt:extract_delta(Msg0), - csrt:accumulate_delta(Delta), - case Msg of - {Ref, {ok, Reply}} -> - Reply; - {Ref, Error} -> - erlang:error(Error); - {rexi_DOWN, Mon, _, Reason} -> - erlang:error({rexi_DOWN, {Node, Reason}}); - Other -> - ?LOG_UNEXPECTED_MSG(Other), - wait_message(Node, Ref, Mon, Timeout) - end. - get_or_create_db(DbName, Options) -> mem3_util:get_or_create_db_int(DbName, Options). diff --git a/src/rexi/src/rexi.erl b/src/rexi/src/rexi.erl index bb7c570375b..a6d933cbe57 100644 --- a/src/rexi/src/rexi.erl +++ b/src/rexi/src/rexi.erl @@ -104,7 +104,8 @@ kill_all(NodeRefs) when is_list(NodeRefs) -> -spec reply(any()) -> any(). reply(Reply) -> {Caller, Ref} = get(rexi_from), - erlang:send(Caller, csrt:maybe_add_delta({Ref, Reply})). + Payload = csrt:maybe_add_delta(Reply), + erlang:send(Caller, {Ref, Payload}). %% Private function used by stream2 to initialize the stream. Message is of the %% form {OriginalRef, {self(),reference()}, Reply}, which enables the @@ -188,7 +189,8 @@ stream2(Msg, Limit, Timeout) -> {ok, Count} -> put(rexi_unacked, Count + 1), {Caller, Ref} = get(rexi_from), - erlang:send(Caller, csrt:maybe_add_delta({Ref, self(), Msg})), + Payload = csrt:maybe_add_delta(Msg), + erlang:send(Caller, {Ref, self(), Payload}), ok catch throw:timeout -> diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index 038612f5a5d..32a739a5d06 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -208,7 +208,8 @@ find_worker(Ref, Tab) -> end. notify_caller({Caller, Ref}, Reason, Delta) -> - Msg = csrt:maybe_add_delta({Ref, {rexi_EXIT, Reason}}, Delta), + Payload = csrt:maybe_add_delta({rexi_EXIT, Reason}, Delta), + Msg = {Ref, Payload}, rexi_utils:send(Caller, Msg). kill_worker(FromRef, #st{clients = Clients} = St) -> diff --git a/src/rexi/src/rexi_utils.erl b/src/rexi/src/rexi_utils.erl index 4ee2586ec7d..beba602e726 100644 --- a/src/rexi/src/rexi_utils.erl +++ b/src/rexi/src/rexi_utils.erl @@ -60,35 +60,39 @@ process_mailbox(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> process_message(RefList, Keypos, Fun, Acc0, TimeoutRef, PerMsgTO) -> receive - Msg -> - process_raw_message(Msg, RefList, Keypos, Fun, Acc0, TimeoutRef) - after PerMsgTO -> - {timeout, Acc0} - end. - -process_raw_message(Payload0, RefList, Keypos, Fun, Acc0, TimeoutRef) -> - {Payload, Delta} = csrt:extract_delta(Payload0), - csrt:accumulate_delta(Delta), - case Payload of {timeout, TimeoutRef} -> {timeout, Acc0}; - {rexi, Ref, Msg} -> + {rexi, Ref, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), case lists:keyfind(Ref, Keypos, RefList) of false -> {ok, Acc0}; Worker -> Fun(Msg, Worker, Acc0) end; - {rexi, Ref, From, Msg} -> + {rexi, Ref, From, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), case lists:keyfind(Ref, Keypos, RefList) of false -> {ok, Acc0}; Worker -> Fun(Msg, {Worker, From}, Acc0) end; + %% Special case for csrt of `{rexi, '$rexi_ping'}` with Delta. + %% Including delta in rexi_ping is essential for getting live info + %% about long running filtered queries that aren't returning rows, as + %% otherwise we won't get the delta until the exhaustion of the find + %% query. + {{rexi, '$rexi_ping'}, {delta, Delta}} -> + csrt:accumulate_delta(Delta), + {ok, Acc0}; {rexi, '$rexi_ping'} -> {ok, Acc0}; - {Ref, Msg} -> + {Ref, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), case lists:keyfind(Ref, Keypos, RefList) of false -> % this was some non-matching message which we will ignore @@ -96,7 +100,9 @@ process_raw_message(Payload0, RefList, Keypos, Fun, Acc0, TimeoutRef) -> Worker -> Fun(Msg, Worker, Acc0) end; - {Ref, From, Msg} -> + {Ref, From, Msg0} -> + {Msg, Delta} = csrt:extract_delta(Msg0), + csrt:accumulate_delta(Delta), case lists:keyfind(Ref, Keypos, RefList) of false -> {ok, Acc0}; @@ -105,4 +111,6 @@ process_raw_message(Payload0, RefList, Keypos, Fun, Acc0, TimeoutRef) -> end; {rexi_DOWN, _, _, _} = Msg -> Fun(Msg, nil, Acc0) + after PerMsgTO -> + {timeout, Acc0} end. diff --git a/src/rexi/test/rexi_tests.erl b/src/rexi/test/rexi_tests.erl index 6a388d16386..a3e73c248b5 100644 --- a/src/rexi/test/rexi_tests.erl +++ b/src/rexi/test/rexi_tests.erl @@ -75,7 +75,7 @@ t_cast(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [potato]}), {Res, Dict} = receive - {Ref, {R, D}, {delta, _}} -> {R, maps:from_list(D)}; + {Ref, {{R, D}, {delta, _}}} -> {R, maps:from_list(D)}; {Ref, {R, D}} -> {R, maps:from_list(D)} end, ?assertEqual(potato, Res), @@ -102,7 +102,7 @@ t_cast_explicit_caller(_) -> end, case csrt:is_enabled() of true -> - ?assertMatch({Ref, {potato, [_ | _]}, {delta, _}}, Result); + ?assertMatch({Ref, {{potato, [_ | _]}, {delta, _}}}, Result); false -> ?assertMatch({Ref, {potato, [_ | _]}}, Result) end. @@ -186,7 +186,7 @@ t_cast_error(_) -> Ref = rexi:cast(node(), self(), {?MODULE, rpc_test_fun, [{error, tomato}]}, []), Res = receive - {Ref, RexiExit, {delta, _}} -> RexiExit; + {Ref, {RexiExit, {delta, _}}} -> RexiExit; {Ref, RexiExit} -> RexiExit end, ?assertMatch({rexi_EXIT, {tomato, [{?MODULE, rpc_test_fun, 1, _} | _]}}, Res). @@ -195,7 +195,7 @@ t_kill(_) -> Ref = rexi:cast(node(), {?MODULE, rpc_test_fun, [{sleep, 10000}]}), WorkerPid = receive - {Ref, {sleeping, Pid}, {delta, _}} -> Pid; + {Ref, {{sleeping, Pid}, {delta, _}}} -> Pid; {Ref, {sleeping, Pid}} -> Pid end, ?assert(is_process_alive(WorkerPid)), @@ -215,14 +215,16 @@ t_ping(_) -> rexi:cast(node(), {?MODULE, rpc_test_fun, [ping]}), Res = receive - {rexi, Ping, {delta, _}} -> Ping; - {rexi, Ping} -> Ping + {{rexi, Ping}, {delta, _}} -> Ping; + {rexi, Ping} -> Ping; + Other -> + Other end, ?assertEqual('$rexi_ping', Res). stream_init(Ref) -> receive - {Ref, From, rexi_STREAM_INIT, {delta, _}} -> + {Ref, From, {rexi_STREAM_INIT, {delta, _}}} -> From; {Ref, From, rexi_STREAM_INIT} -> From @@ -230,8 +232,8 @@ stream_init(Ref) -> recv(Ref) when is_reference(Ref) -> receive - {Ref, _, Msg, {delta, _}} -> Msg; - {Ref, Msg, {delta, _}} -> Msg; + {Ref, _, {Msg, {delta, _}}} -> Msg; + {Ref, {Msg, {delta, _}}} -> Msg; {Ref, _, Msg} -> Msg; {Ref, Msg} -> Msg after 500 -> timeout From 089f02d24adaf63d996ba93fa374ef3200753642 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Thu, 8 May 2025 16:14:45 -0700 Subject: [PATCH 16/54] More cleanup --- src/couch_stats/src/csrt.erl | 38 ++++++++++++++---------------------- 1 file changed, 15 insertions(+), 23 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 534696d93af..b979530d14c 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -124,10 +124,6 @@ destroy_pid_ref(_PidRef) -> %% Context lifecycle API %% -%% TODO: shouldn't need this? -%% create_resource(#rctx{} = Rctx) -> -%% csrt_server:create_resource(Rctx). - -spec create_worker_context(From, MFA, Nonce) -> pid_ref() | false when From :: pid_ref(), MFA :: mfa(), Nonce :: nonce(). create_worker_context(From, {M, F, _A}, Nonce) -> @@ -154,8 +150,6 @@ create_coordinator_context(#httpd{method = Verb, nonce = Nonce}, Path0) -> -spec create_context(Type :: rctx_type(), Nonce :: term()) -> pid_ref(). create_context(Type, Nonce) -> Rctx = csrt_server:new_context(Type, Nonce), - %% TODO: which approach - %% PidRef = csrt_server:pid_ref(Rctx), PidRef = get_pid_ref(Rctx), set_pid_ref(PidRef), csrt_util:set_delta_zero(Rctx), @@ -203,12 +197,20 @@ set_context_handler_fun(Mod, Func) when update_handler_fun(_, _, undefined) -> false; update_handler_fun(Mod, Func, PidRef) -> - Rctx = get_resource(PidRef), - %% TODO: #coordinator{} assumption needs to adapt for other types - #coordinator{} = Coordinator0 = csrt_server:get_context_type(Rctx), - Coordinator = Coordinator0#coordinator{mod = Mod, func = Func}, - csrt_server:set_context_type(Coordinator, PidRef), - ok. + case get_resource(PidRef) of + undefined -> + ok; + #rctx{} = Rctx -> + %% TODO: #coordinator{} assumption needs to adapt for other types + case csrt_server:get_context_type(Rctx) of + #coordinator{} = Coordinator0 -> + Coordinator = Coordinator0#coordinator{mod = Mod, func = Func}, + csrt_server:set_context_type(Coordinator, PidRef), + ok; + _ -> + ok + end + end. %% @equiv set_context_username(User, get_pid_ref()) set_context_username(User) -> @@ -335,14 +337,6 @@ make_delta() -> rctx_delta(TA, TB) -> csrt_util:rctx_delta(TA, TB). -%% TODO: cleanup return type -%%-spec update_counter(Field :: rctx_field(), Count :: non_neg_integer()) -> false | ok. -%%-spec update_counter(Field :: non_neg_integer(), Count :: non_neg_integer()) -> false | ok. -%%update_counter(_Field, Count) when Count < 0 -> -%% false; -%%update_counter(Field, Count) when Count >= 0 -> -%% is_enabled() andalso csrt_server:update_counter(get_pid_ref(), Field, Count). - %% %% aggregate query api %% @@ -351,8 +345,6 @@ rctx_delta(TA, TB) -> active() -> csrt_query:active(). -%% TODO: ensure Type fields align with type specs -%%-spec active(Type :: rctx_type()) -> [rctx()]. -spec active(Type :: json) -> [rctx()]. active(Type) -> csrt_query:active(Type). @@ -428,7 +420,7 @@ maybe_add_delta(T, Delta) -> csrt_util:maybe_add_delta(T, Delta). %% -%% Internal Operations assuming is_enabled() == true +%% Tests %% -ifdef(TEST). From a9990332da8e5885535d919072180c72874d8071 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Thu, 8 May 2025 16:39:44 -0700 Subject: [PATCH 17/54] Erlfmt rexi_tests.erl --- src/rexi/test/rexi_tests.erl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rexi/test/rexi_tests.erl b/src/rexi/test/rexi_tests.erl index a3e73c248b5..d156d2c514d 100644 --- a/src/rexi/test/rexi_tests.erl +++ b/src/rexi/test/rexi_tests.erl @@ -217,8 +217,7 @@ t_ping(_) -> receive {{rexi, Ping}, {delta, _}} -> Ping; {rexi, Ping} -> Ping; - Other -> - Other + Other -> Other end, ?assertEqual('$rexi_ping', Res). From e070513c5525e186d7842ff57037591c19152429 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 16 May 2025 19:35:09 -0700 Subject: [PATCH 18/54] Revert "CI Bump.." This reverts commit 020743fe6793b5abb5bbefb14fdfd63ccc45afda. --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 45c5eb8adae..ece0ca891d1 100644 --- a/README.rst +++ b/README.rst @@ -1,7 +1,6 @@ Apache CouchDB README ===================== - +---------+ | |1| |2| | +---------+ From befdfcb7537aa2804ad0cee52c5f1331aed385a1 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 16 May 2025 19:41:22 -0700 Subject: [PATCH 19/54] More PR cleanup --- rel/overlay/etc/default.ini | 25 +++++++++++++------------ src/couch_stats/src/couch_stats.erl | 2 -- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index a8663e3580e..a53049f48f3 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1124,9 +1124,10 @@ url = {{nouveau_url}} [csrt] enabled = true -; CSRT Rexi Server Init P tracking +; CSRT Rexi Server RPC Worker Spawn Tracking ; Enable these to enable additional metrics for RPC worker spawn rates -; Mod and Function are separated by double underscores +; measuring how often RPC workers are spawned by way of rexi_server:init_p. +; Mod and Function are separated by double underscores. [csrt.init_p] enabled = false fabric_rpc__all_docs = true @@ -1138,14 +1139,14 @@ fabric_rpc__open_doc = true fabric_rpc__update_docs = true fabric_rpc__open_shard = true -;; CSRT dbname matchers -;; Given a dbname and a positive integer, this will enable an IO matcher -;; against the provided db for any requests that induce IO in quantities -;; greater than the provided threshold on any one of: ioq_calls, rows_read -;; docs_read, get_kp_node, get_kv_node, or changes_processed. -;; +; CSRT dbname matchers +; Given a dbname and a positive integer, this will enable an IO matcher +; against the provided db for any requests that induce IO in quantities +; greater than the provided threshold on any one of: ioq_calls, rows_read +; docs_read, get_kp_node, get_kv_node, or changes_processed. +; [csrt_logger.dbnames_io] -;; foo = 100 -;; _dbs = 123 -;; _users = 234 -;; foo/bar = 200 +; foo = 100 +; _dbs = 123 +; _users = 234 +; foo/bar = 200 diff --git a/src/couch_stats/src/couch_stats.erl b/src/couch_stats/src/couch_stats.erl index f92950fa940..2426773f848 100644 --- a/src/couch_stats/src/couch_stats.erl +++ b/src/couch_stats/src/couch_stats.erl @@ -57,7 +57,6 @@ increment_counter(Name, Value) -> %% Should maybe_track_local happen before or after notify? %% If after, only currently tracked metrics declared in the app's %% stats_description.cfg will be trackable locally. Pros/cons. - %io:format("NOTIFY_EXISTING_METRIC: ~p || ~p || ~p~n", [Name, Op, Type]), ok = maybe_track_local_counter(Name, Value), case couch_stats_util:get_counter(Name, stats()) of {ok, Ctx} -> couch_stats_counter:increment(Ctx, Value); @@ -113,7 +112,6 @@ now_sec() -> %% Only potentially track positive increments to counters -spec maybe_track_local_counter(any(), any()) -> ok. maybe_track_local_counter(Name, Val) when is_integer(Val) andalso Val > 0 -> - %%io:format("maybe_track_local[~p]: ~p~n", [Val, Name]), csrt:maybe_inc(Name, Val), ok; maybe_track_local_counter(_, _) -> From 11f97550403399466637492a247136a10910f9ca Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 2 Jun 2025 15:01:02 -0700 Subject: [PATCH 20/54] More cleanup --- rel/overlay/etc/default.ini | 2 ++ src/couch_stats/src/csrt.erl | 20 ++++++++++++++------ src/couch_stats/src/csrt_logger.erl | 24 ++++++++++++++++++------ src/couch_stats/src/csrt_server.erl | 14 +++++++++----- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index a53049f48f3..f1fbcf11c82 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1123,6 +1123,8 @@ url = {{nouveau_url}} ; Couch Stats Resource Tracker (CSRT) [csrt] enabled = true +;enabled = false +;should_truncate_reports = false ; CSRT Rexi Server RPC Worker Spawn Tracking ; Enable these to enable additional metrics for RPC worker spawn rates diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index b979530d14c..49dbcd35045 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -147,16 +147,24 @@ create_coordinator_context(#httpd{method = Verb, nonce = Nonce}, Path0) -> false end. --spec create_context(Type :: rctx_type(), Nonce :: term()) -> pid_ref(). +-spec create_context(Type :: rctx_type(), Nonce :: term()) -> pid_ref() | false. create_context(Type, Nonce) -> Rctx = csrt_server:new_context(Type, Nonce), PidRef = get_pid_ref(Rctx), set_pid_ref(PidRef), - csrt_util:set_delta_zero(Rctx), - csrt_util:set_delta_a(Rctx), - csrt_server:create_resource(Rctx), - csrt_logger:track(Rctx), - PidRef. + try + csrt_util:set_delta_zero(Rctx), + csrt_util:set_delta_a(Rctx), + csrt_server:create_resource(Rctx), + csrt_logger:track(Rctx), + PidRef + catch + _:_ -> + csrt_server:destroy_resource(Rctx), + %% destroy_context(PidRef) clears the tracker too + destroy_context(PidRef), + false + end. -spec set_context_dbname(DbName :: binary()) -> boolean(). set_context_dbname(DbName) -> diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index a4b28043e29..fff724084b4 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -28,9 +28,11 @@ do_lifetime_report/1, do_status_report/1, do_report/2, - maybe_report/2 + maybe_report/2, + should_truncate_reports/0 ]). +%% gen_server callbacks -export([ start_link/0, init/1, @@ -172,6 +174,10 @@ maybe_report(ReportName, PidRef) -> ok end. +-spec should_truncate_reports() -> boolean(). +should_truncate_reports() -> + config:get_boolean(?CSRT, "should_truncate_reports", false). + -spec do_lifetime_report(Rctx :: rctx()) -> boolean(). do_lifetime_report(Rctx) -> do_report("csrt-pid-usage-lifetime", Rctx). @@ -182,7 +188,13 @@ do_status_report(Rctx) -> -spec do_report(ReportName :: string(), Rctx :: rctx()) -> boolean(). do_report(ReportName, #rctx{} = Rctx) -> - couch_log:report(ReportName, csrt_util:to_json(Rctx)). + JRctx = case {should_truncate_reports(), csrt_util:to_json(Rctx)} of + {true, JRctx0} -> + maps:filter(fun(_K, V) -> V > 0 end, JRctx0); + {false, JRctx0} -> + JRctx0 + end, + couch_log:report(ReportName, JRctx). %% %% Process lifetime logging api @@ -337,10 +349,10 @@ set_matchers_term(Matchers) when is_map(Matchers) -> -spec initialize_matchers() -> ok. initialize_matchers() -> DefaultMatchers = [ - {docs_read, fun matcher_on_docs_read/1, 100}, - {dbname, fun matcher_on_dbname/1, <<"foo">>}, - {rows_read, fun matcher_on_rows_read/1, 100}, - {docs_written, fun matcher_on_docs_written/1, 1}, + {docs_read, fun matcher_on_docs_read/1, 1000}, + %%{dbname, fun matcher_on_dbname/1, <<"foo">>}, + {rows_read, fun matcher_on_rows_read/1, 1000}, + {docs_written, fun matcher_on_docs_written/1, 500}, %%{view_rows_read, fun matcher_on_rows_read/1, 1000}, %%{slow_reqs, fun matcher_on_slow_reqs/1, 10000}, {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index fa22558ba38..b1945a4756c 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -124,7 +124,13 @@ get_resource(PidRef) -> match_resource(undefined) -> []; match_resource(#rctx{} = Rctx) -> - ets:match_object(?CSRT_ETS, Rctx). + try + ets:match_object(?CSRT_ETS, Rctx) + catch + _:_ -> + [] + end. + -spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). is_rctx_field(Field) -> @@ -188,8 +194,7 @@ init([]) -> ets:new(?CSRT_ETS, [ named_table, public, - {decentralized_counters, true}, - {write_concurrency, true}, + {write_concurrency, auto}, {read_concurrency, true}, {keypos, #rctx.pid_ref} ]), @@ -209,5 +214,4 @@ handle_cast(_Msg, State) -> update_element(undefined, _Update) -> false; update_element({_Pid, _Ref} = PidRef, Update) -> - %% TODO: should we take any action when the update fails? - catch ets:update_element(?CSRT_ETS, PidRef, Update). + (catch ets:update_element(?CSRT_ETS, PidRef, Update)) == true. From 6fd6a29a307648c883d152eace63a49a92e110ae Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 2 Jun 2025 18:37:54 -0700 Subject: [PATCH 21/54] Cleanup #rctx{} and other reworkings --- src/chttpd/src/chttpd.erl | 6 +- src/couch/priv/stats_descriptions.cfg | 9 --- src/couch/src/couch_os_process.erl | 11 +-- .../src/couch_stats_resource_tracker.hrl | 64 ++++++++---------- src/couch_stats/src/csrt.erl | 67 +++++++++---------- src/couch_stats/src/csrt_logger.erl | 49 +++++++++----- src/couch_stats/src/csrt_query.erl | 5 -- src/couch_stats/src/csrt_server.erl | 33 +++++---- src/couch_stats/src/csrt_util.erl | 36 +++++----- .../test/eunit/csrt_logger_tests.erl | 32 ++++++--- .../test/eunit/csrt_server_tests.erl | 8 +-- src/fabric/src/fabric_rpc.erl | 4 +- 12 files changed, 163 insertions(+), 161 deletions(-) diff --git a/src/chttpd/src/chttpd.erl b/src/chttpd/src/chttpd.erl index e9631ce48f3..7a94233f659 100644 --- a/src/chttpd/src/chttpd.erl +++ b/src/chttpd/src/chttpd.erl @@ -341,7 +341,7 @@ handle_request_int(MochiReq) -> %% This is probably better in before_request, but having Path is nice csrt:create_coordinator_context(HttpReq0, Path), - csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), + csrt:set_context_handler_fun({?MODULE, ?FUNCTION_NAME}), {HttpReq2, Response} = case before_request(HttpReq0) of @@ -373,7 +373,7 @@ handle_request_int(MochiReq) -> before_request(HttpReq) -> try - csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), + csrt:set_context_handler_fun({?MODULE, ?FUNCTION_NAME}), chttpd_stats:init(), chttpd_plugin:before_request(HttpReq) catch @@ -407,7 +407,7 @@ process_request(#httpd{mochi_req = MochiReq} = HttpReq) -> RawUri = MochiReq:get(raw_path), try - csrt:set_context_handler_fun(?MODULE, ?FUNCTION_NAME), + csrt:set_context_handler_fun({?MODULE, ?FUNCTION_NAME}), couch_httpd:validate_host(HttpReq), check_request_uri_length(RawUri), check_url_encoding(RawUri), diff --git a/src/couch/priv/stats_descriptions.cfg b/src/couch/priv/stats_descriptions.cfg index 586c6e66a7a..ade882c07c7 100644 --- a/src/couch/priv/stats_descriptions.cfg +++ b/src/couch/priv/stats_descriptions.cfg @@ -430,10 +430,6 @@ {type, counter}, {desc, <<"number of legacy checksums found in couch_file instances">>} ]}. -{[couchdb, btree, folds], [ - {type, counter}, - {desc, <<"number of couch btree kv fold callback invocations">>} -]}. {[couchdb, btree, get_node, kp_node], [ {type, counter}, {desc, <<"number of couch btree kp_nodes read">>} @@ -450,11 +446,6 @@ {type, counter}, {desc, <<"number of couch btree kv_nodes written">>} ]}. -%% CSRT (couch_stats_resource_tracker) stats -{[couchdb, csrt, delta_missing_t0], [ - {type, counter}, - {desc, <<"number of csrt contexts without a proper startime">>} -]}. {[pread, exceed_eof], [ {type, counter}, {desc, <<"number of the attempts to read beyond end of db file">>} diff --git a/src/couch/src/couch_os_process.erl b/src/couch/src/couch_os_process.erl index 1a018bd0417..59ceeca13a1 100644 --- a/src/couch/src/couch_os_process.erl +++ b/src/couch/src/couch_os_process.erl @@ -246,9 +246,8 @@ bump_cmd_time_stat(Cmd, USec) when is_list(Cmd), is_integer(USec) -> bump_time_stat(ddoc_new, USec); [<<"ddoc">>, _, [<<"validate_doc_update">> | _] | _] -> bump_time_stat(ddoc_vdu, USec); - [<<"ddoc">>, _, [<<"filters">> | _], [Docs | _] | _] -> - bump_time_stat(ddoc_filter, USec), - bump_volume_stat(ddoc_filter, Docs); + [<<"ddoc">>, _, [<<"filters">> | _] | _] -> + bump_time_stat(ddoc_filter, USec); [<<"ddoc">> | _] -> bump_time_stat(ddoc_other, USec); _ -> @@ -259,12 +258,6 @@ bump_time_stat(Stat, USec) when is_atom(Stat), is_integer(USec) -> couch_stats:increment_counter([couchdb, query_server, calls, Stat]), couch_stats:increment_counter([couchdb, query_server, time, Stat], USec). -bump_volume_stat(ddoc_filter = Stat, Docs) when is_atom(Stat), is_list(Docs) -> - couch_stats:increment_counter([couchdb, query_server, volume, Stat], length(Docs)); -bump_volume_stat(_, _) -> - %% TODO: handle other stats? - ok. - log_level("debug") -> debug; log_level("info") -> diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index c37c910ba6e..17f8c0a8467 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -20,56 +20,54 @@ -define(PID_REF, csrt_pid_ref). %% track local ID -define(TRACKER_PID, csrt_tracker). %% tracker pid --define(MANGO_EVAL_MATCH, mango_eval_match). -define(DB_OPEN_DOC, docs_read). -define(DB_OPEN, db_open). -define(COUCH_SERVER_OPEN, db_open). -define(COUCH_BT_GET_KP_NODE, get_kp_node). -define(COUCH_BT_GET_KV_NODE, get_kv_node). --define(COUCH_BT_WRITE_KP_NODE, write_kp_node). --define(COUCH_BT_WRITE_KV_NODE, write_kv_node). +%% "Example to extend CSRT" +%%-define(COUCH_BT_WRITE_KP_NODE, write_kp_node). +%%-define(COUCH_BT_WRITE_KV_NODE, write_kv_node). -define(COUCH_JS_FILTER, js_filter). -define(COUCH_JS_FILTERED_DOCS, js_filtered_docs). -define(IOQ_CALLS, ioq_calls). -define(DOCS_WRITTEN, docs_written). -define(ROWS_READ, rows_read). -%% TODO: use dedicated changes_processed or use rows_read? -%%-define(FRPC_CHANGES_PROCESSED, rows_read). --define(FRPC_CHANGES_PROCESSED, changes_processed). -define(FRPC_CHANGES_RETURNED, changes_returned). -define(STATS_TO_KEYS, #{ - [mango, evaluate_selector] => ?MANGO_EVAL_MATCH, [couchdb, database_reads] => ?DB_OPEN_DOC, - [fabric_rpc, changes, processed] => ?FRPC_CHANGES_PROCESSED, + %% Double on ?ROWS_READ for changes_processed as we only need the one + %% field, as opposed to needing both metrics to distinguish changes + %% workloads and view/_all_docs. + [fabric_rpc, changes, processed] => ?ROWS_READ, [fabric_rpc, changes, returned] => ?FRPC_CHANGES_RETURNED, [fabric_rpc, view, rows_read] => ?ROWS_READ, [couchdb, couch_server, open] => ?DB_OPEN, [couchdb, btree, get_node, kp_node] => ?COUCH_BT_GET_KP_NODE, - [couchdb, btree, get_node, kv_node] => ?COUCH_BT_GET_KV_NODE, + [couchdb, btree, get_node, kv_node] => ?COUCH_BT_GET_KV_NODE %% NOTE: these stats are not local to the RPC worker, need forwarding - [couchdb, btree, write_node, kp_node] => ?COUCH_BT_WRITE_KP_NODE, - [couchdb, btree, write_node, kv_node] => ?COUCH_BT_WRITE_KV_NODE, - [couchdb, query_server, calls, ddoc_filter] => ?COUCH_JS_FILTER, - [couchdb, query_server, volume, ddoc_filter] => ?COUCH_JS_FILTERED_DOCS + %% "Example to extend CSRT" + %% [couchdb, btree, write_node, kp_node] => ?COUCH_BT_WRITE_KP_NODE, + %% [couchdb, btree, write_node, kv_node] => ?COUCH_BT_WRITE_KV_NODE, + %% [couchdb, query_server, calls, ddoc_filter] => ?COUCH_JS_FILTER }). -define(KEYS_TO_FIELDS, #{ ?DB_OPEN => #rctx.?DB_OPEN, ?ROWS_READ => #rctx.?ROWS_READ, ?FRPC_CHANGES_RETURNED => #rctx.?FRPC_CHANGES_RETURNED, - ?FRPC_CHANGES_PROCESSED => #rctx.?FRPC_CHANGES_PROCESSED, ?DOCS_WRITTEN => #rctx.?DOCS_WRITTEN, ?IOQ_CALLS => #rctx.?IOQ_CALLS, ?COUCH_JS_FILTER => #rctx.?COUCH_JS_FILTER, ?COUCH_JS_FILTERED_DOCS => #rctx.?COUCH_JS_FILTERED_DOCS, - ?MANGO_EVAL_MATCH => #rctx.?MANGO_EVAL_MATCH, ?DB_OPEN_DOC => #rctx.?DB_OPEN_DOC, ?COUCH_BT_GET_KP_NODE => #rctx.?COUCH_BT_GET_KP_NODE, - ?COUCH_BT_GET_KV_NODE => #rctx.?COUCH_BT_GET_KV_NODE, - ?COUCH_BT_WRITE_KP_NODE => #rctx.?COUCH_BT_WRITE_KP_NODE, - ?COUCH_BT_WRITE_KV_NODE => #rctx.?COUCH_BT_WRITE_KV_NODE + ?COUCH_BT_GET_KV_NODE => #rctx.?COUCH_BT_GET_KV_NODE + %% "Example to extend CSRT" + %% ?COUCH_BT_WRITE_KP_NODE => #rctx.?COUCH_BT_WRITE_KP_NODE, + %% ?COUCH_BT_WRITE_KV_NODE => #rctx.?COUCH_BT_WRITE_KV_NODE }). -type throw(_Reason) :: no_return(). @@ -110,21 +108,17 @@ docs_read = 0 :: non_neg_integer(), docs_written = 0 :: non_neg_integer(), rows_read = 0 :: non_neg_integer(), - changes_processed = 0 :: non_neg_integer(), changes_returned = 0 :: non_neg_integer(), ioq_calls = 0 :: non_neg_integer(), - io_bytes_read = 0 :: non_neg_integer(), - io_bytes_written = 0 :: non_neg_integer(), - js_evals = 0 :: non_neg_integer(), js_filter = 0 :: non_neg_integer(), js_filtered_docs = 0 :: non_neg_integer(), - mango_eval_match = 0 :: non_neg_integer(), %% TODO: switch record definitions to be macro based, eg: %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer(), get_kv_node = 0 :: non_neg_integer(), - get_kp_node = 0 :: non_neg_integer(), - write_kv_node = 0 :: non_neg_integer(), - write_kp_node = 0 :: non_neg_integer() + get_kp_node = 0 :: non_neg_integer() + %% Add these later + %%write_kv_node = 0 :: non_neg_integer(), + %%write_kp_node = 0 :: non_neg_integer() }). -type rctx_field() :: @@ -139,26 +133,22 @@ | docs_read | docs_written | rows_read - | changes_processed | changes_returned | ioq_calls - | io_bytes_read - | io_bytes_written - | js_evals | js_filter | js_filtered_docs - | mango_eval_match | get_kv_node - | get_kp_node - | write_kv_node - | write_kp_node. + | get_kp_node. + %%| write_kv_node + %%| write_kp_node. + -type coordinator_rctx() :: #rctx{type :: coordinator()}. -type rpc_worker_rctx() :: #rctx{type :: rpc_worker()}. -type rctx() :: #rctx{} | coordinator_rctx() | rpc_worker_rctx(). -type maybe_rctx() :: rctx() | undefined. -%% TODO: solidify nonce type +%% TODO: solidify nonce type and ideally move to couch_db.hrl -type nonce() :: any(). -type dbname() :: iodata(). -type username() :: iodata(). @@ -167,8 +157,8 @@ -type maybe_delta() :: delta() | undefined. -type tagged_delta() :: {delta, maybe_delta()}. --type matcher_name() :: string(). %% TODO: switch to string to allow dynamic options +-type matcher_name() :: string(). -type matcher() :: {ets:match_spec(), ets:comp_match_spec()}. --type maybe_matcher() :: matcher() | undefined. -type matchers() :: #{matcher_name() => matcher()} | #{}. +-type maybe_matcher() :: matcher() | undefined. -type maybe_matchers() :: matchers() | undefined. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 49dbcd35045..26e11382468 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -171,14 +171,22 @@ set_context_dbname(DbName) -> set_context_dbname(DbName, get_pid_ref()). -spec set_context_dbname(DbName, PidRef) -> boolean() when - DbName :: binary(), PidRef :: pid_ref() | undefined. + DbName :: binary(), PidRef :: maybe_pid_ref(). set_context_dbname(_, undefined) -> false; set_context_dbname(DbName, PidRef) -> is_enabled() andalso csrt_server:set_context_dbname(DbName, PidRef). --spec set_context_handler_fun(Fun :: function()) -> boolean(). -set_context_handler_fun(Fun) when is_function(Fun) -> +-spec set_context_handler_fun(Handler) -> boolean() when + Handler :: function() | {atom(), atom()}. +set_context_handler_fun(Handler) -> + set_context_handler_fun(Handler, get_pid_ref()). + +-spec set_context_handler_fun(Handler, PidRef) -> boolean() when + Handler :: function() | {atom(), atom()}, PidRef :: maybe_pid_ref(). +set_context_handler_fun(_, undefined) -> + false; +set_context_handler_fun(Fun, PidRef) when is_function(Fun) -> case is_enabled() of false -> false; @@ -186,38 +194,14 @@ set_context_handler_fun(Fun) when is_function(Fun) -> FProps = erlang:fun_info(Fun), Mod = proplists:get_value(module, FProps), Func = proplists:get_value(name, FProps), - update_handler_fun(Mod, Func, get_pid_ref()) - end. - --spec set_context_handler_fun(Mod :: atom(), Func :: atom()) -> boolean(). -set_context_handler_fun(Mod, Func) when - is_atom(Mod) andalso is_atom(Func) --> + set_context_handler_fun({Mod, Func}, PidRef) + end; +set_context_handler_fun({Mod, Func}, PidRef) -> case is_enabled() of false -> false; true -> - update_handler_fun(Mod, Func, get_pid_ref()) - end. - --spec update_handler_fun(Mod, Func, PidRef) -> boolean() when - Mod :: atom(), Func :: atom(), PidRef :: maybe_pid_ref(). -update_handler_fun(_, _, undefined) -> - false; -update_handler_fun(Mod, Func, PidRef) -> - case get_resource(PidRef) of - undefined -> - ok; - #rctx{} = Rctx -> - %% TODO: #coordinator{} assumption needs to adapt for other types - case csrt_server:get_context_type(Rctx) of - #coordinator{} = Coordinator0 -> - Coordinator = Coordinator0#coordinator{mod = Mod, func = Func}, - csrt_server:set_context_type(Coordinator, PidRef), - ok; - _ -> - ok - end + csrt_server:set_context_handler_fun({Mod, Func}, PidRef) end. %% @equiv set_context_username(User, get_pid_ref()) @@ -456,11 +440,26 @@ teardown(Ctx) -> test_util:stop_couch(Ctx). t_static_map_translations(_) -> - ?assert(lists:all(fun(E) -> maps:is_key(E, ?KEYS_TO_FIELDS) end, maps:values(?STATS_TO_KEYS))), + %% Bit of a hack to delete duplicated rows_read between views and changes + SingularStats = lists:delete(rows_read, maps:values(?STATS_TO_KEYS)), + ?assert(lists:all(fun(E) -> maps:is_key(E, ?KEYS_TO_FIELDS) end, SingularStats)), %% TODO: properly handle ioq_calls field ?assertEqual( - lists:sort(maps:values(?STATS_TO_KEYS)), - lists:delete(docs_written, lists:delete(ioq_calls, lists:sort(maps:keys(?KEYS_TO_FIELDS)))) + lists:sort(SingularStats), + lists:sort(lists:foldl( + fun(E, A) -> + %% Ignore fields regarding external processes + Deletions = [docs_written, ioq_calls, js_filter, js_filtered_docs], + case lists:member(E, Deletions) of + true -> + A; + false -> + [E | A] + end + end, + [], + maps:keys(?KEYS_TO_FIELDS) + )) ). t_should_track_init_p(_) -> diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index fff724084b4..0a0d5706101 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -50,6 +50,7 @@ %% Matchers -export([ + deregister_matcher/1, get_matcher/1, get_matchers/0, is_match/1, @@ -74,7 +75,7 @@ -define(CONF_MATCHERS_DBNAMES, "csrt_logger.dbnames_io"). -record(st, { - matchers = #{} + registered_matchers = #{} }). -spec track(Rctx :: rctx()) -> pid(). @@ -117,6 +118,10 @@ tracker({Pid, _Ref} = PidRef) -> register_matcher(Name, MSpec) -> gen_server:call(?MODULE, {register, Name, MSpec}). +-spec deregister_matcher(Name :: string()) -> ok. +deregister_matcher(Name) -> + gen_server:call(?MODULE, {deregister, Name}). + -spec log_process_lifetime_report(PidRef :: pid_ref()) -> ok. log_process_lifetime_report(PidRef) -> case csrt_util:is_enabled() of @@ -223,21 +228,22 @@ start_link() -> gen_server:start_link({local, ?MODULE}, ?MODULE, [], []). init([]) -> - ok = initialize_matchers(), + St = #st{}, + ok = initialize_matchers(St#st.registered_matchers), ok = subscribe_changes(), - {ok, #st{}}. + {ok, St}. -handle_call({register, Name, MSpec}, _From, #st{matchers = Matchers} = St) -> - case add_matcher(Name, MSpec, Matchers) of - {ok, Matchers1} -> - set_matchers_term(Matchers1), - {reply, ok, St#st{matchers = Matchers1}}; +handle_call({register, Name, MSpec}, _From, #st{registered_matchers = RMatchers} = St) -> + case add_matcher(Name, MSpec, RMatchers) of + {ok, RMatchers1} -> + ok = initialize_matchers(RMatchers1), + {reply, ok, St#st{registered_matchers = RMatchers1}}; {error, badarg} = Error -> {reply, Error, St} end; handle_call(reload_matchers, _From, St) -> couch_log:warning("Reloading persistent term matchers", []), - ok = initialize_matchers(), + ok = initialize_matchers(St#st.registered_matchers), {reply, ok, St}; handle_call(_, _From, State) -> {reply, ok, State}. @@ -273,13 +279,12 @@ matcher_on_dbname_io_threshold(DbName, Threshold) when get_kv_node = KVN, get_kp_node = KPN, docs_read = Docs, - rows_read = Rows, - changes_processed = Chgs + rows_read = Rows } = R ) when DbName =:= DbName1 andalso ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or - (Rows >= Threshold) or (Chgs >= Threshold)) + (Rows >= Threshold)) -> R end). @@ -315,7 +320,7 @@ matcher_on_worker_changes_processed(Threshold) when ets:fun2ms( fun( #rctx{ - changes_processed = Processed, + rows_read = Processed, changes_returned = Returned } = R ) when (Processed - Returned) >= Threshold -> @@ -344,10 +349,11 @@ add_matcher(Name, MSpec, Matchers) -> -spec set_matchers_term(Matchers :: matchers()) -> ok. set_matchers_term(Matchers) when is_map(Matchers) -> - persistent_term:put({?MODULE, all_csrt_matchers}, Matchers). + persistent_term:put(?MATCHERS_KEY, Matchers). --spec initialize_matchers() -> ok. -initialize_matchers() -> +-spec initialize_matchers(RegisteredMatchers :: map()) -> ok. +initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> + %% Standard matchers to conditionally enable DefaultMatchers = [ {docs_read, fun matcher_on_docs_read/1, 1000}, %%{dbname, fun matcher_on_dbname/1, <<"foo">>}, @@ -358,6 +364,8 @@ initialize_matchers() -> {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, {ioq_calls, fun matcher_on_ioq_calls/1, 10000} ], + + %% Add enabled Matchers for standard matchers Matchers = lists:foldl( fun({Name0, MatchGenFunc, Threshold0}, Matchers0) when is_atom(Name0) -> Name = atom_to_list(Name0), @@ -381,6 +389,8 @@ initialize_matchers() -> #{}, DefaultMatchers ), + + %% Add additional dbname_io matchers Matchers1 = lists:foldl( fun({Dbname, Value}, Matchers0) -> try list_to_integer(Value) of @@ -410,8 +420,11 @@ initialize_matchers() -> config:get(?CONF_MATCHERS_DBNAMES) ), - couch_log:notice("Initialized ~p CSRT Logger matchers", [maps:size(Matchers1)]), - persistent_term:put(?MATCHERS_KEY, Matchers1), + %% Finally, merge in the dynamically registered matchers, with priority + Matchers2 = maps:merge(Matchers1, RegisteredMatchers), + + couch_log:notice("Initialized ~p CSRT Logger matchers", [maps:size(Matchers2)]), + set_matchers_term(Matchers2), ok. -spec matcher_enabled(Name :: string()) -> boolean(). diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl index 1708ebebc34..9e6dc6e2eec 100644 --- a/src/couch_stats/src/csrt_query.erl +++ b/src/couch_stats/src/csrt_query.erl @@ -102,15 +102,10 @@ field(#rctx{username = Val}, username) -> Val; field(#rctx{db_open = Val}, db_open) -> Val; field(#rctx{docs_read = Val}, docs_read) -> Val; field(#rctx{rows_read = Val}, rows_read) -> Val; -field(#rctx{changes_processed = Val}, changes_processed) -> Val; field(#rctx{changes_returned = Val}, changes_returned) -> Val; field(#rctx{ioq_calls = Val}, ioq_calls) -> Val; -field(#rctx{io_bytes_read = Val}, io_bytes_read) -> Val; -field(#rctx{io_bytes_written = Val}, io_bytes_written) -> Val; -field(#rctx{js_evals = Val}, js_evals) -> Val; field(#rctx{js_filter = Val}, js_filter) -> Val; field(#rctx{js_filtered_docs = Val}, js_filtered_docs) -> Val; -field(#rctx{mango_eval_match = Val}, mango_eval_match) -> Val; field(#rctx{get_kv_node = Val}, get_kv_node) -> Val; field(#rctx{get_kp_node = Val}, get_kp_node) -> Val. diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index b1945a4756c..963ef3aba3e 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -32,8 +32,9 @@ match_resource/1, new_context/2, set_context_dbname/2, - set_context_username/2, + set_context_handler_fun/2, set_context_type/2, + set_context_username/2, update_counter/3 ]). @@ -70,20 +71,29 @@ set_context_dbname(_, undefined) -> set_context_dbname(DbName, PidRef) -> update_element(PidRef, [{#rctx.dbname, DbName}]). -%%set_context_handler_fun(_, undefined) -> -%% ok; -%%set_context_handler_fun(Fun, PidRef) when is_function(Fun) -> -%% FProps = erlang:fun_info(Fun), -%% Mod = proplists:get_value(module, FProps), -%% Func = proplists:get_value(name, FProps), -%% #rctx{type=#coordinator{}=Coordinator} = get_resource(PidRef), -%% Update = [{#rctx.type, Coordinator#coordinator{mod=Mod, func=Func}}], -%% update_element(PidRef, Update). +-spec set_context_handler_fun({Mod, Func}, PidRef) -> boolean() when + Mod :: atom(), Func :: atom(), PidRef :: maybe_pid_ref(). +set_context_handler_fun(_, undefined) -> + false; +set_context_handler_fun({Mod, Func}, PidRef) -> + case get_resource(PidRef) of + undefined -> + false; + #rctx{} = Rctx -> + %% TODO: #coordinator{} assumption needs to adapt for other types + case csrt_server:get_context_type(Rctx) of + #coordinator{} = Coordinator0 -> + Coordinator = Coordinator0#coordinator{mod = Mod, func = Func}, + set_context_type(Coordinator, PidRef); + _ -> + false + end + end. -spec set_context_username(UserName, PidRef) -> boolean() when UserName :: username(), PidRef :: maybe_pid_ref(). set_context_username(_, undefined) -> - ok; + false; set_context_username(UserName, PidRef) -> update_element(PidRef, [{#rctx.username, UserName}]). @@ -131,7 +141,6 @@ match_resource(#rctx{} = Rctx) -> [] end. - -spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). is_rctx_field(Field) -> maps:is_key(Field, ?KEYS_TO_FIELDS). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 8c773a44f1e..9f3ffc1b04d 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -185,10 +185,10 @@ to_json(#rctx{} = Rctx) -> type => convert_type(Rctx#rctx.type), get_kp_node => Rctx#rctx.get_kp_node, get_kv_node => Rctx#rctx.get_kv_node, - write_kp_node => Rctx#rctx.write_kp_node, - write_kv_node => Rctx#rctx.write_kv_node, + %% "Example to extend CSRT" + %% write_kp_node => Rctx#rctx.write_kp_node, + %% write_kv_node => Rctx#rctx.write_kv_node, changes_returned => Rctx#rctx.changes_returned, - changes_processed => Rctx#rctx.changes_processed, ioq_calls => Rctx#rctx.ioq_calls }. @@ -228,16 +228,18 @@ map_to_rctx_field(get_kp_node, Val, Rctx) -> Rctx#rctx{get_kp_node = Val}; map_to_rctx_field(get_kv_node, Val, Rctx) -> Rctx#rctx{get_kv_node = Val}; -map_to_rctx_field(write_kp_node, Val, Rctx) -> - Rctx#rctx{write_kp_node = Val}; -map_to_rctx_field(write_kv_node, Val, Rctx) -> - Rctx#rctx{write_kv_node = Val}; +%% "Example to extend CSRT" +%% map_to_rctx_field(write_kp_node, Val, Rctx) -> +%% Rctx#rctx{write_kp_node = Val}; +%% map_to_rctx_field(write_kv_node, Val, Rctx) -> +%% Rctx#rctx{write_kv_node = Val}; map_to_rctx_field(changes_returned, Val, Rctx) -> Rctx#rctx{changes_returned = Val}; -map_to_rctx_field(changes_processed, Val, Rctx) -> - Rctx#rctx{changes_processed = Val}; map_to_rctx_field(ioq_calls, Val, Rctx) -> - Rctx#rctx{ioq_calls = Val}. + Rctx#rctx{ioq_calls = Val}; +map_to_rctx_field(_, _, Rctx) -> + %% Unknown key, could throw but just move on + Rctx. -spec field(Field :: rctx_field(), Rctx :: rctx()) -> any(). field(updated_at, #rctx{updated_at = Val}) -> @@ -272,11 +274,10 @@ field(get_kv_node, #rctx{get_kv_node = Val}) -> Val; field(changes_returned, #rctx{changes_returned = Val}) -> Val; -field(changes_processed, #rctx{changes_processed = Val}) -> - Val; field(ioq_calls, #rctx{ioq_calls = Val}) -> Val. +-spec add_delta(T :: term(), Delta :: maybe_delta()) -> term(). add_delta(T, {delta, undefined}) -> T; add_delta(T, {delta, _} = Delta) -> @@ -284,6 +285,7 @@ add_delta(T, {delta, _} = Delta) -> add_delta(T, _Delta) -> T. +-spec extract_delta(T :: term()) -> {term(), maybe_delta()}. extract_delta({Msg, {delta, Delta}}) -> {Msg, Delta}; extract_delta(Msg) -> @@ -293,6 +295,7 @@ extract_delta(Msg) -> get_delta(PidRef) -> {delta, make_delta(PidRef)}. +-spec maybe_add_delta(T :: term()) -> term(). maybe_add_delta(T) -> case is_enabled() of false -> @@ -303,6 +306,7 @@ maybe_add_delta(T) -> %% Allow for externally provided Delta in error handling scenarios %% eg in cases like rexi_server:notify_caller/3 +-spec maybe_add_delta(T :: term(), Delta :: maybe_delta()) -> term(). maybe_add_delta(T, Delta) -> case is_enabled() of false -> @@ -311,6 +315,7 @@ maybe_add_delta(T, Delta) -> maybe_add_delta_int(T, Delta) end. +-spec maybe_add_delta_int(T :: term(), Delta :: maybe_delta()) -> term(). maybe_add_delta_int(T, undefined) -> T; maybe_add_delta_int(T, {delta, undefined}) -> @@ -339,7 +344,6 @@ rctx_delta(#rctx{} = TA, #rctx{} = TB) -> js_filtered_docs => TB#rctx.js_filtered_docs - TA#rctx.js_filtered_docs, rows_read => TB#rctx.rows_read - TA#rctx.rows_read, changes_returned => TB#rctx.changes_returned - TA#rctx.changes_returned, - changes_processed => TB#rctx.changes_processed - TA#rctx.changes_processed, get_kp_node => TB#rctx.get_kp_node - TA#rctx.get_kp_node, get_kv_node => TB#rctx.get_kv_node - TA#rctx.get_kv_node, db_open => TB#rctx.db_open - TA#rctx.db_open, @@ -373,11 +377,11 @@ set_delta_zero(TZ) -> get_pid_ref() -> get(?PID_REF). --spec get_pid_ref(Rctx :: rctx()) -> pid_ref(). +-spec get_pid_ref(Rctx :: rctx()) -> maybe_pid_ref(). get_pid_ref(#rctx{pid_ref = PidRef}) -> PidRef; -get_pid_ref(R) -> - throw({unexpected, R}). +get_pid_ref(_) -> + undefined. -spec set_pid_ref(PidRef :: pid_ref()) -> pid_ref(). set_pid_ref(PidRef) -> diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 7bb3bcbeb0e..66485fcd6de 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -27,6 +27,7 @@ -define(THRESHOLD_DBNAME, <<"foo">>). -define(THRESHOLD_DBNAME_IO, 91). -define(THRESHOLD_DOCS_READ, 123). +-define(THRESHOLD_DOCS_WRITTEN, 12). -define(THRESHOLD_IOQ_CALLS, 439). -define(THRESHOLD_ROWS_READ, 143). -define(THRESHOLD_CHANGES, 79). @@ -49,7 +50,7 @@ csrt_logger_matchers_test_() -> fun setup/0, fun teardown/1, [ - ?TDEF_FE(t_matcher_on_dbname), + %%?TDEF_FE(t_matcher_on_dbname), %% TODO: add back in or delete ?TDEF_FE(t_matcher_on_dbnames_io), ?TDEF_FE(t_matcher_on_docs_read), ?TDEF_FE(t_matcher_on_docs_written), @@ -92,6 +93,9 @@ setup() -> ok = config:set( "csrt_logger.matchers_threshold", "docs_read", integer_to_list(?THRESHOLD_DOCS_READ), false ), + ok = config:set( + "csrt_logger.matchers_threshold", "docs_written", integer_to_list(?THRESHOLD_DOCS_WRITTEN), false + ), ok = config:set( "csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false ), @@ -156,15 +160,16 @@ rctx_gen(Opts0) -> csrt_util:map_to_rctx( maps:fold( fun - %% Hack for changes because we need to modify both changes_processed - %% and changes_returned but the latter must be <= the former + %% Hack for changes because we need to modify both + %% changes_processed (rows_read) and changes_returned but the + %% latter must be <= the former ('_do_changes', V, Acc) -> case V of true -> Processed = R(), Returned = (one_of([0, 0, 1, Processed, rand:uniform(Processed)]))(), maps:put( - changes_processed, + rows_read, Processed, maps:put(changes_returned, Returned, Acc) ); @@ -238,10 +243,11 @@ t_matcher_on_docs_read(#{rctxs := Rctxs0}) -> ). t_matcher_on_docs_written(#{rctxs := Rctxs0}) -> + Threshold = ?THRESHOLD_DOCS_WRITTEN, %% Make sure we have at least one match - Rctxs = [rctx_gen(#{docs_written => 73}) | Rctxs0], + Rctxs = [rctx_gen(#{docs_written => Threshold + 10}) | Rctxs0], ?assertEqual( - lists:sort(lists:filter(matcher_gte(docs_written, 1), Rctxs)), + lists:sort(lists:filter(matcher_gte(docs_written, Threshold), Rctxs)), lists:sort(lists:filter(matcher_for_csrt("docs_written"), Rctxs)), "Docs written matcher" ). @@ -259,10 +265,10 @@ t_matcher_on_rows_read(#{rctxs := Rctxs0}) -> t_matcher_on_worker_changes_processed(#{rctxs := Rctxs0}) -> Threshold = ?THRESHOLD_CHANGES, %% Make sure we have at least one match - Rctxs = [rctx_gen(#{changes_processed => Threshold + 10}) | Rctxs0], + Rctxs = [rctx_gen(#{rows_read => Threshold + 10}) | Rctxs0], ChangesFilter = fun(R) -> Ret = csrt_util:field(changes_returned, R), - Proc = csrt_util:field(changes_processed, R), + Proc = csrt_util:field(rows_read, R), (Proc - Ret) >= Threshold end, ?assertEqual( @@ -346,13 +352,19 @@ matcher_for(Field, Value, Op) -> matcher_for_csrt(MatcherName) -> Matchers = #{MatcherName => {_, _} = csrt_logger:get_matcher(MatcherName)}, - fun(Rctx) -> csrt_logger:is_match(Rctx, Matchers) end. + case csrt_logger:get_matcher(MatcherName) of + {_, _} = Matcher -> + Matchers = #{MatcherName => Matcher}, + fun(Rctx) -> csrt_logger:is_match(Rctx, Matchers) end; + _ -> + throw({missing_matcher, MatcherName}) + end. matcher_for_dbname_io(Dbname0, Threshold) -> Dbname = list_to_binary(Dbname0), fun(Rctx) -> DbnameA = csrt_util:field(dbname, Rctx), - Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read, changes_processed], + Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read], Vals = [{F, csrt_util:field(F, Rctx)} || F <- Fields], Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun(V) -> V >= Threshold end, Vals) end. diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index fa39c82e561..e90f0783a8b 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -316,7 +316,7 @@ t_changes({_Ctx, DbName, View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => ?DB_Q, - changes_processed => docs_count(View), + rows_read => docs_count(View), changes_returned => docs_count(View), docs_read => 0, docs_written => 0, @@ -338,7 +338,7 @@ t_changes_limit_zero({_Ctx, DbName, _View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => ?DB_Q, - changes_processed => assert_gte(?DB_Q), + rows_read => assert_gte(?DB_Q), changes_returned => assert_gte(?DB_Q), docs_read => 0, docs_written => 0, @@ -370,7 +370,7 @@ t_changes_js_filtered({_Ctx, DbName, {DDocId, _ViewName} = View}) -> ok = rctx_assert(Rctx, #{ nonce => Nonce, db_open => assert_gte(?DB_Q), - changes_processed => assert_gte(docs_count(View)), + rows_read => assert_gte(docs_count(View)), changes_returned => round(?DOCS_COUNT / 2), docs_read => assert_gte(docs_count(View)), docs_written => 0, @@ -468,8 +468,6 @@ rctx_assert(Rctx, Asserts0) -> changes_returned => 0, js_filter => 0, js_filtered_docs => 0, - write_kp_node => 0, - write_kv_node => 0, nonce => undefined, db_open => 0, rows_read => 0, diff --git a/src/fabric/src/fabric_rpc.erl b/src/fabric/src/fabric_rpc.erl index afa1656009f..3c0e8e2ed4d 100644 --- a/src/fabric/src/fabric_rpc.erl +++ b/src/fabric/src/fabric_rpc.erl @@ -494,11 +494,8 @@ view_cb({meta, Meta}, Acc) -> ok = rexi:stream2({meta, Meta}), {ok, Acc}; view_cb({row, Props}, #mrargs{extra = Options} = Acc) -> - %% TODO: distinguish between rows and docs - %% TODO: wire in csrt tracking %% TODO: distinguish between all_docs vs view call couch_stats:increment_counter([fabric_rpc, view, rows_read]), - %%csrt:inc(rows_read), % Adding another row ViewRow = fabric_view_row:from_props(Props, Options), ok = rexi:stream2(ViewRow), @@ -518,6 +515,7 @@ reduce_cb({meta, Meta}, Acc, _Options) -> {ok, Acc}; reduce_cb({row, Props}, Acc, Options) -> % Adding another row + couch_stats:increment_counter([fabric_rpc, view, rows_read]), ViewRow = fabric_view_row:from_props(Props, Options), ok = rexi:stream2(ViewRow), {ok, Acc}; From fbd745558a3fec51470cecc53beb8a6dc1d37a1f Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 2 Jun 2025 18:41:46 -0700 Subject: [PATCH 22/54] make erlfmt-format --- src/couch_stats/src/csrt.erl | 30 ++++++++++--------- src/couch_stats/src/csrt_logger.erl | 13 ++++---- .../test/eunit/csrt_logger_tests.erl | 5 +++- 3 files changed, 27 insertions(+), 21 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 26e11382468..2e9cdd1e5f5 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -446,20 +446,22 @@ t_static_map_translations(_) -> %% TODO: properly handle ioq_calls field ?assertEqual( lists:sort(SingularStats), - lists:sort(lists:foldl( - fun(E, A) -> - %% Ignore fields regarding external processes - Deletions = [docs_written, ioq_calls, js_filter, js_filtered_docs], - case lists:member(E, Deletions) of - true -> - A; - false -> - [E | A] - end - end, - [], - maps:keys(?KEYS_TO_FIELDS) - )) + lists:sort( + lists:foldl( + fun(E, A) -> + %% Ignore fields regarding external processes + Deletions = [docs_written, ioq_calls, js_filter, js_filtered_docs], + case lists:member(E, Deletions) of + true -> + A; + false -> + [E | A] + end + end, + [], + maps:keys(?KEYS_TO_FIELDS) + ) + ) ). t_should_track_init_p(_) -> diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 0a0d5706101..94d5044fed2 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -193,12 +193,13 @@ do_status_report(Rctx) -> -spec do_report(ReportName :: string(), Rctx :: rctx()) -> boolean(). do_report(ReportName, #rctx{} = Rctx) -> - JRctx = case {should_truncate_reports(), csrt_util:to_json(Rctx)} of - {true, JRctx0} -> - maps:filter(fun(_K, V) -> V > 0 end, JRctx0); - {false, JRctx0} -> - JRctx0 - end, + JRctx = + case {should_truncate_reports(), csrt_util:to_json(Rctx)} of + {true, JRctx0} -> + maps:filter(fun(_K, V) -> V > 0 end, JRctx0); + {false, JRctx0} -> + JRctx0 + end, couch_log:report(ReportName, JRctx). %% diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 66485fcd6de..c4f32775490 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -94,7 +94,10 @@ setup() -> "csrt_logger.matchers_threshold", "docs_read", integer_to_list(?THRESHOLD_DOCS_READ), false ), ok = config:set( - "csrt_logger.matchers_threshold", "docs_written", integer_to_list(?THRESHOLD_DOCS_WRITTEN), false + "csrt_logger.matchers_threshold", + "docs_written", + integer_to_list(?THRESHOLD_DOCS_WRITTEN), + false ), ok = config:set( "csrt_logger.matchers_threshold", "ioq_calls", integer_to_list(?THRESHOLD_IOQ_CALLS), false From 04c588d8ea94b5c8c84c9cd34190b7261c838095 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 3 Jun 2025 15:39:18 -0700 Subject: [PATCH 23/54] Cleanup csrt_query:field/2 --- src/couch_stats/src/csrt_query.erl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl index 9e6dc6e2eec..a813b4c4594 100644 --- a/src/couch_stats/src/csrt_query.erl +++ b/src/couch_stats/src/csrt_query.erl @@ -93,21 +93,21 @@ field(#rctx{pid_ref = Val}, pid_ref) -> Val; %% structure provided by the caller of *_by aggregator functions below. %% For now, we just always return jiffy:encode'able data types. field(#rctx{nonce = Val}, nonce) -> Val; -%%field(#rctx{from=Val}, from) -> Val; -%% TODO: fix this, perhaps move it all to csrt_util? field(#rctx{type = Val}, type) -> csrt_util:convert_type(Val); field(#rctx{dbname = Val}, dbname) -> Val; field(#rctx{username = Val}, username) -> Val; -%%field(#rctx{path=Val}, path) -> Val; field(#rctx{db_open = Val}, db_open) -> Val; field(#rctx{docs_read = Val}, docs_read) -> Val; +field(#rctx{docs_written = Val}, docs_written) -> Val; field(#rctx{rows_read = Val}, rows_read) -> Val; field(#rctx{changes_returned = Val}, changes_returned) -> Val; field(#rctx{ioq_calls = Val}, ioq_calls) -> Val; field(#rctx{js_filter = Val}, js_filter) -> Val; field(#rctx{js_filtered_docs = Val}, js_filtered_docs) -> Val; field(#rctx{get_kv_node = Val}, get_kv_node) -> Val; -field(#rctx{get_kp_node = Val}, get_kp_node) -> Val. +field(#rctx{get_kp_node = Val}, get_kp_node) -> Val; +field(#rctx{started_at = Val}, started_at) -> Val; +field(#rctx{updated_at = Val}, updated_at) -> Val. curry_field(Field) -> fun(Ele) -> field(Ele, Field) end. From 1063b8acf0a70d909fe3e3f40dbd2e5714d28a00 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 3 Jun 2025 16:53:23 -0700 Subject: [PATCH 24/54] Clarify is_rctx_stat_field --- .../src/couch_stats_resource_tracker.hrl | 6 +++++- src/couch_stats/src/csrt.erl | 4 ++-- src/couch_stats/src/csrt_server.erl | 21 ++++++++++--------- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 17f8c0a8467..97e38781cce 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -35,6 +35,8 @@ -define(ROWS_READ, rows_read). -define(FRPC_CHANGES_RETURNED, changes_returned). +%% Mapping of couch_stat metric names to #rctx{} field names. +%% These are used for fields that we inc a counter on. -define(STATS_TO_KEYS, #{ [couchdb, database_reads] => ?DB_OPEN_DOC, %% Double on ?ROWS_READ for changes_processed as we only need the one @@ -54,7 +56,9 @@ %% [couchdb, query_server, calls, ddoc_filter] => ?COUCH_JS_FILTER }). --define(KEYS_TO_FIELDS, #{ +%% Mapping of stat field names to their corresponding record entries. +%% This only includes integer fields valid for ets:update_counter +-define(STAT_KEYS_TO_FIELDS, #{ ?DB_OPEN => #rctx.?DB_OPEN, ?ROWS_READ => #rctx.?ROWS_READ, ?FRPC_CHANGES_RETURNED => #rctx.?FRPC_CHANGES_RETURNED, diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 2e9cdd1e5f5..6116a976ef4 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -442,7 +442,7 @@ teardown(Ctx) -> t_static_map_translations(_) -> %% Bit of a hack to delete duplicated rows_read between views and changes SingularStats = lists:delete(rows_read, maps:values(?STATS_TO_KEYS)), - ?assert(lists:all(fun(E) -> maps:is_key(E, ?KEYS_TO_FIELDS) end, SingularStats)), + ?assert(lists:all(fun(E) -> maps:is_key(E, ?STAT_KEYS_TO_FIELDS) end, SingularStats)), %% TODO: properly handle ioq_calls field ?assertEqual( lists:sort(SingularStats), @@ -459,7 +459,7 @@ t_static_map_translations(_) -> end end, [], - maps:keys(?KEYS_TO_FIELDS) + maps:keys(?STAT_KEYS_TO_FIELDS) ) ) ). diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index 963ef3aba3e..4181bbb6947 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -141,15 +141,17 @@ match_resource(#rctx{} = Rctx) -> [] end. --spec is_rctx_field(Field :: rctx_field() | atom()) -> boolean(). -is_rctx_field(Field) -> - maps:is_key(Field, ?KEYS_TO_FIELDS). +%% Is this a valid #rctx{} field for inducing ets:update_counter upon? +-spec is_rctx_stat_field(Field :: rctx_field() | atom()) -> boolean(). +is_rctx_stat_field(Field) -> + maps:is_key(Field, ?STAT_KEYS_TO_FIELDS). --spec get_rctx_field(Field :: rctx_field()) -> +%% Get the #rctx{} field record index of the corresponding stat counter field +-spec get_rctx_stat_field(Field :: rctx_field()) -> non_neg_integer() | throw({badkey, Key :: any()}). -get_rctx_field(Field) -> - maps:get(Field, ?KEYS_TO_FIELDS). +get_rctx_stat_field(Field) -> + maps:get(Field, ?STAT_KEYS_TO_FIELDS). -spec update_counter(PidRef, Field, Count) -> non_neg_integer() when PidRef :: maybe_pid_ref(), @@ -158,10 +160,9 @@ get_rctx_field(Field) -> update_counter(undefined, _Field, _Count) -> 0; update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> - %% TODO: mem3 crashes without catch, why do we lose the stats table? - case is_rctx_field(Field) of + case is_rctx_stat_field(Field) of true -> - Update = {get_rctx_field(Field), Count}, + Update = {get_rctx_stat_field(Field), Count}, try ets:update_counter(?CSRT_ETS, PidRef, Update, #rctx{pid_ref = PidRef}) catch @@ -185,7 +186,7 @@ inc(undefined, _Field, _) -> inc(_PidRef, _Field, 0) -> 0; inc({_Pid, _Ref} = PidRef, Field, N) when is_integer(N) andalso N > 0 -> - case is_rctx_field(Field) of + case is_rctx_stat_field(Field) of true -> update_counter(PidRef, Field, N); false -> From f6a7bf6e1b6839a5cfaf3fae521babd9a26536c8 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 3 Jun 2025 17:16:21 -0700 Subject: [PATCH 25/54] Fix csrt:inc/N typespec --- src/couch_stats/src/csrt.erl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 6116a976ef4..bd076b5de63 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -272,11 +272,21 @@ do_report(ReportName, PidRef) -> -spec inc(Key :: rctx_field()) -> non_neg_integer(). inc(Key) -> - is_enabled() andalso csrt_server:inc(get_pid_ref(), Key). + case is_enabled() of + true -> + csrt_server:inc(get_pid_ref(), Key); + false -> + 0 + end. -spec inc(Key :: rctx_field(), N :: non_neg_integer()) -> non_neg_integer(). inc(Key, N) when is_integer(N) andalso N >= 0 -> - is_enabled() andalso csrt_server:inc(get_pid_ref(), Key, N). + case is_enabled() of + true -> + csrt_server:inc(get_pid_ref(), Key, N); + false -> + 0 + end. -spec maybe_inc(Stat :: atom(), Val :: non_neg_integer()) -> non_neg_integer(). maybe_inc(Stat, Val) -> From d973007f8de04a82320c391b870369089e8dadab Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 9 Jun 2025 18:21:45 -0700 Subject: [PATCH 26/54] Batch accumulate_delta updates in single ets:update_counter call --- src/couch_stats/src/csrt.erl | 3 ++- src/couch_stats/src/csrt_logger.erl | 3 ++- src/couch_stats/src/csrt_server.erl | 39 ++++++++++++++++++++++++++++- 3 files changed, 42 insertions(+), 3 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index bd076b5de63..272a107eba2 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -321,7 +321,8 @@ docs_written(N) -> accumulate_delta(Delta) when is_map(Delta) -> %% TODO: switch to creating a batch of updates to invoke a single %% update_counter rather than sequentially invoking it for each field - is_enabled() andalso maps:foreach(fun inc/2, Delta), + %%is_enabled() andalso maps:foreach(fun inc/2, Delta), + is_enabled() andalso csrt_server:update_counters(get_pid_ref(), Delta), ok; accumulate_delta(undefined) -> ok. diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 94d5044fed2..4f54edeae73 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -179,9 +179,10 @@ maybe_report(ReportName, PidRef) -> ok end. +%% Whether or not to remove zero value fields from reports -spec should_truncate_reports() -> boolean(). should_truncate_reports() -> - config:get_boolean(?CSRT, "should_truncate_reports", false). + config:get_boolean(?CSRT, "should_truncate_reports", true). -spec do_lifetime_report(Rctx :: rctx()) -> boolean(). do_lifetime_report(Rctx) -> diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index 4181bbb6947..8448041b6ad 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -35,7 +35,8 @@ set_context_handler_fun/2, set_context_type/2, set_context_username/2, - update_counter/3 + update_counter/3, + update_counters/2 ]). -include_lib("stdlib/include/ms_transform.hrl"). @@ -173,6 +174,42 @@ update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> 0 end. +-spec update_counters(PidRef, Delta) -> boolean() when + PidRef :: maybe_pid_ref(), + Delta :: delta(). +update_counters(undefined, _Delta) -> + false; +update_counters({_Pid, _Ref} = PidRef, Delta) when is_map(Delta) -> + Updates = maps:fold( + fun(Field, Count, Acc) -> + case is_rctx_stat_field(Field) of + true -> + [{get_rctx_stat_field(Field), Count} | Acc]; + false -> + %% This skips entries that are not is_rctx_stat_field's + %% Another approach would be: + %% lists:all(lists:map(fun is_rctx_stat_field/1, maps:keys(Delta))) + %% But that's a lot of looping for not even acumulating the update. + Acc + end + end, + [], + Delta + ), + + case Updates of + [] -> + false; + _ -> + try + ets:update_counter(?CSRT_ETS, PidRef, Updates, #rctx{pid_ref = PidRef}), + true + catch + _:_ -> + false + end + end. + -spec inc(PidRef :: maybe_pid_ref(), Field :: rctx_field()) -> non_neg_integer(). inc(PidRef, Field) -> inc(PidRef, Field, 1). From 7211093d2d65f671e7601cce7f67dc1d829d929e Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 9 Jun 2025 18:32:35 -0700 Subject: [PATCH 27/54] Fixup csrt_logger report tests --- src/couch_stats/test/eunit/csrt_logger_tests.erl | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index c4f32775490..26d1929cb5c 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -192,8 +192,17 @@ rctx_gen(Opts0) -> rctxs() -> [rctx_gen() || _ <- lists:seq(1, ?RCTX_COUNT)]. -t_do_report(#{rctx := Rctx}) -> +jrctx(Rctx) -> JRctx = csrt_util:to_json(Rctx), + case csrt_logger:should_truncate_reports() of + true -> + maps:filter(fun(_K, V) -> V > 0 end, JRctx); + false -> + JRctx + end. + +t_do_report(#{rctx := Rctx}) -> + JRctx = jrctx(Rctx), ReportName = "foo", ?assert(csrt_logger:do_report(ReportName, Rctx), "CSRT _logger:do_report " ++ ReportName), ?assert(meck:validate(couch_log), "CSRT do_report"), @@ -204,7 +213,7 @@ t_do_report(#{rctx := Rctx}) -> ). t_do_lifetime_report(#{rctx := Rctx}) -> - JRctx = csrt_util:to_json(Rctx), + JRctx = jrctx(Rctx), ReportName = "csrt-pid-usage-lifetime", ?assert( csrt_logger:do_lifetime_report(Rctx), @@ -217,7 +226,7 @@ t_do_lifetime_report(#{rctx := Rctx}) -> ). t_do_status_report(#{rctx := Rctx}) -> - JRctx = csrt_util:to_json(Rctx), + JRctx = jrctx(Rctx), ReportName = "csrt-pid-usage-status", ?assert(csrt_logger:do_status_report(Rctx), "csrt_logger:do_ " ++ ReportName), ?assert(meck:validate(couch_log), "CSRT validate couch_log"), From a7f3342bf38f3a4f35f2f3f10699d55c6e91e0e6 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 10 Jun 2025 17:57:49 -0700 Subject: [PATCH 28/54] Update deregister logic and testing --- .../src/couch_stats_resource_tracker.hrl | 2 + src/couch_stats/src/csrt.erl | 3 - src/couch_stats/src/csrt_logger.erl | 38 ++++++++++- .../test/eunit/csrt_logger_tests.erl | 63 ++++++++++++++++++- 4 files changed, 98 insertions(+), 8 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 97e38781cce..bd2df44f96c 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -150,6 +150,7 @@ -type coordinator_rctx() :: #rctx{type :: coordinator()}. -type rpc_worker_rctx() :: #rctx{type :: rpc_worker()}. -type rctx() :: #rctx{} | coordinator_rctx() | rpc_worker_rctx(). +-type rctxs() :: [#rctx{}] | []. -type maybe_rctx() :: rctx() | undefined. %% TODO: solidify nonce type and ideally move to couch_db.hrl @@ -164,5 +165,6 @@ -type matcher_name() :: string(). -type matcher() :: {ets:match_spec(), ets:comp_match_spec()}. -type matchers() :: #{matcher_name() => matcher()} | #{}. +-type matcher_matches() :: #{matcher_name() => rctxs()} | #{}. -type maybe_matcher() :: matcher() | undefined. -type maybe_matchers() :: matchers() | undefined. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 272a107eba2..2551b14c7f7 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -319,9 +319,6 @@ docs_written(N) -> -spec accumulate_delta(Delta :: map() | undefined) -> ok. accumulate_delta(Delta) when is_map(Delta) -> - %% TODO: switch to creating a batch of updates to invoke a single - %% update_counter rather than sequentially invoking it for each field - %%is_enabled() andalso maps:foreach(fun inc/2, Delta), is_enabled() andalso csrt_server:update_counters(get_pid_ref(), Delta), ok; accumulate_delta(undefined) -> diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 4f54edeae73..a75d9ef7026 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -51,8 +51,11 @@ %% Matchers -export([ deregister_matcher/1, + find_all_matches/2, + find_matches/2, get_matcher/1, get_matchers/0, + get_registered_matchers/0, is_match/1, is_match/2, matcher_on_dbname/1, @@ -131,12 +134,27 @@ log_process_lifetime_report(PidRef) -> ok end. -%% TODO: add Matchers spec +%% Return a subset of Matchers for each Matcher that matches on Rctxs -spec find_matches(Rctxs :: [rctx()], Matchers :: matchers()) -> matchers(). find_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> maps:filter( fun(_Name, {_MSpec, CompMSpec}) -> - catch [] =/= ets:match_spec_run(Rctxs, CompMSpec) + (catch ets:match_spec_run(Rctxs, CompMSpec)) =/= [] + end, + Matchers + ). + +%% Return a Map of #{MatcherName => SRctxs :: rctxs()} for all MatcherName => Matcher +%% in Matchers where SRctxs is the subset of Rctxs matched by the given Matcher +-spec find_all_matches(Rctxs :: rctxs(), Matchers :: matchers()) -> matcher_matches(). +find_all_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> + maps:map( + fun(_Name, {_MSpec, CompMSpec}) -> + try + ets:match_spec_run(Rctxs, CompMSpec) + catch _:_ -> + [] + end end, Matchers ). @@ -153,13 +171,16 @@ get_matchers() -> get_matcher(Name) -> maps:get(Name, get_matchers(), undefined). +-spec get_registered_matchers() -> matchers(). +get_registered_matchers() -> + gen_server:call(?MODULE, get_registered_matchers, infinity). + -spec is_match(Rctx :: maybe_rctx()) -> boolean(). is_match(undefined) -> false; is_match(#rctx{} = Rctx) -> is_match(Rctx, get_matchers()). -%% TODO: add Matchers spec -spec is_match(Rctx :: maybe_rctx(), Matchers :: matchers()) -> boolean(). is_match(undefined, _Matchers) -> false; @@ -243,10 +264,21 @@ handle_call({register, Name, MSpec}, _From, #st{registered_matchers = RMatchers} {error, badarg} = Error -> {reply, Error, St} end; +handle_call({deregister, Name}, _From, #st{registered_matchers = RMatchers} = St) -> + case maps:is_key(Name, RMatchers) of + false -> + {reply, {error, missing_matcher}, St}; + true -> + RMatchers1 = maps:remove(Name, RMatchers), + ok = initialize_matchers(RMatchers1), + {reply, ok, St#st{registered_matchers = RMatchers1}} + end; handle_call(reload_matchers, _From, St) -> couch_log:warning("Reloading persistent term matchers", []), ok = initialize_matchers(St#st.registered_matchers), {reply, ok, St}; +handle_call(get_registered_matchers, _From, St) -> + {reply, St#st.registered_matchers, St}; handle_call(_, _From, State) -> {reply, ok, State}. diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 26d1929cb5c..b229c62c86e 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -32,7 +32,7 @@ -define(THRESHOLD_ROWS_READ, 143). -define(THRESHOLD_CHANGES, 79). -csrt_logger_works_test_() -> +csrt_logger_reporting_works_test_() -> { foreach, fun setup_reporting/0, @@ -57,7 +57,8 @@ csrt_logger_matchers_test_() -> ?TDEF_FE(t_matcher_on_rows_read), ?TDEF_FE(t_matcher_on_worker_changes_processed), ?TDEF_FE(t_matcher_on_ioq_calls), - ?TDEF_FE(t_matcher_on_nonce) + ?TDEF_FE(t_matcher_on_nonce), + ?TDEF_FE(t_matcher_register_deregister) ] }. @@ -343,6 +344,64 @@ t_matcher_on_dbnames_io(#{rctxs := Rctxs0}) -> "dbname_io foo/bar matcher" ). +t_matcher_register_deregister(#{rctxs := Rctxs0}) -> + CrazyDbName = <<"asdf123@?!&#fdsa">>, + MName = "Crazy-Matcher", + MSpec = csrt_logger:matcher_on_dbname(CrazyDbName), + %% Add an extra Rctx with CrazyDbName to create a specific match + ExtraRctx = rctx_gen(#{dbname => CrazyDbName}), + %% Make sure we have at least one match + Rctxs = [ExtraRctx | Rctxs0], + + ?assertEqual(#{}, csrt_logger:get_registered_matchers(), "no current registered matchers"), + ?assertEqual(ok, csrt_logger:register_matcher(MName, MSpec), "register matcher"), + %% TODO: use test_wait thing until initialize_matchers rebuilds + CompMSpec = test_util:wait( + fun() -> + case csrt_logger:get_matcher(MName) of + undefined -> + wait; + {MSpec, _Ref} = CompMSpec0 -> + CompMSpec0 + end + end + ), + Matchers = #{MName => CompMSpec}, + ?assert(CompMSpec =/= timeout, "newly registered matcher was initialized"), + ?assertEqual([MName], maps:keys(csrt_logger:get_registered_matchers()), "correct current registered matchers"), + ?assert(csrt_logger:is_match(ExtraRctx, Matchers), "our registered matcher matches expectedly"), + ?assert(csrt_logger:is_match(ExtraRctx), "our registered matcher is picked up and matches expectedly"), + ?assertEqual( + Matchers, + csrt_logger:find_matches(Rctxs, Matchers), + "we find our matcher and no extra matchers" + ), + ?assert( + maps:is_key( + MName, + csrt_logger:find_matches(Rctxs, csrt_logger:get_matchers()) + ), + "find our CrazyDbName matcher in matches against all registered matchers" + ), + ?assertEqual( + #{MName => [ExtraRctx]}, + csrt_logger:find_all_matches(Rctxs, Matchers), + "find our CrazyDb ExtraRctx with our Matcher, and nothing else" + ), + ?assertEqual(ok, csrt_logger:deregister_matcher(MName), "deregister_matcher returns ok"), + Matcher2 = test_util:wait( + fun() -> + case csrt_logger:get_matcher(MName) of + undefined -> + undefined; + _ -> + wait + end + end + ), + ?assertEqual(undefined, Matcher2, "matcher was deregistered successfully"), + ?assertEqual(#{}, csrt_logger:get_registered_matchers(), "no leftover registered matchers"). + load_rctx(PidRef) -> %% Add slight delay to accumulate RPC response deltas timer:sleep(50), From b58e04aa48f70920b69bfa000137bb0144615d8e Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 10 Jun 2025 18:04:12 -0700 Subject: [PATCH 29/54] make erlfmt-format --- src/couch_stats/src/csrt_logger.erl | 5 +++-- src/couch_stats/test/eunit/csrt_logger_tests.erl | 14 ++++++++++---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index a75d9ef7026..ff43e09febc 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -152,8 +152,9 @@ find_all_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) - fun(_Name, {_MSpec, CompMSpec}) -> try ets:match_spec_run(Rctxs, CompMSpec) - catch _:_ -> - [] + catch + _:_ -> + [] end end, Matchers diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index b229c62c86e..b4bb5c5aba8 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -355,7 +355,6 @@ t_matcher_register_deregister(#{rctxs := Rctxs0}) -> ?assertEqual(#{}, csrt_logger:get_registered_matchers(), "no current registered matchers"), ?assertEqual(ok, csrt_logger:register_matcher(MName, MSpec), "register matcher"), - %% TODO: use test_wait thing until initialize_matchers rebuilds CompMSpec = test_util:wait( fun() -> case csrt_logger:get_matcher(MName) of @@ -368,9 +367,16 @@ t_matcher_register_deregister(#{rctxs := Rctxs0}) -> ), Matchers = #{MName => CompMSpec}, ?assert(CompMSpec =/= timeout, "newly registered matcher was initialized"), - ?assertEqual([MName], maps:keys(csrt_logger:get_registered_matchers()), "correct current registered matchers"), + ?assertEqual( + [MName], + maps:keys(csrt_logger:get_registered_matchers()), + "correct current registered matchers" + ), ?assert(csrt_logger:is_match(ExtraRctx, Matchers), "our registered matcher matches expectedly"), - ?assert(csrt_logger:is_match(ExtraRctx), "our registered matcher is picked up and matches expectedly"), + ?assert( + csrt_logger:is_match(ExtraRctx), + "our registered matcher is picked up and matches expectedly" + ), ?assertEqual( Matchers, csrt_logger:find_matches(Rctxs, Matchers), @@ -384,7 +390,7 @@ t_matcher_register_deregister(#{rctxs := Rctxs0}) -> "find our CrazyDbName matcher in matches against all registered matchers" ), ?assertEqual( - #{MName => [ExtraRctx]}, + #{MName => [ExtraRctx]}, csrt_logger:find_all_matches(Rctxs, Matchers), "find our CrazyDb ExtraRctx with our Matcher, and nothing else" ), From c9371eee37fea0f88de5bcff311bd761bf4a33e5 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 10 Jun 2025 18:12:13 -0700 Subject: [PATCH 30/54] Assert registered matchers persist a after global reload --- src/couch_stats/test/eunit/csrt_logger_tests.erl | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index b4bb5c5aba8..beab30f4e84 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -394,6 +394,19 @@ t_matcher_register_deregister(#{rctxs := Rctxs0}) -> csrt_logger:find_all_matches(Rctxs, Matchers), "find our CrazyDb ExtraRctx with our Matcher, and nothing else" ), + ?assertEqual(ok, csrt_logger:reload_matchers(), "we can reload matchers"), + ?assertEqual( + [MName], + maps:keys(csrt_logger:get_registered_matchers()), + "correct current registered matchers after a global reload" + ), + ?assert( + maps:is_key( + MName, + csrt_logger:find_matches(Rctxs, csrt_logger:get_matchers()) + ), + "our matcher still behaves expectedly after a global matcher reload" + ), ?assertEqual(ok, csrt_logger:deregister_matcher(MName), "deregister_matcher returns ok"), Matcher2 = test_util:wait( fun() -> From 06da5f2ecfe852a3dcc3d7103d842c9f29c43a8e Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 11 Jun 2025 17:53:45 -0700 Subject: [PATCH 31/54] Cleanup is_enabled settings --- rel/overlay/etc/default.ini | 15 +++++++++++---- .../src/couch_stats_resource_tracker.hrl | 3 ++- src/couch_stats/src/csrt.erl | 4 ++-- src/couch_stats/src/csrt_logger.erl | 5 +++-- src/couch_stats/src/csrt_util.erl | 18 +++++++++++++----- 5 files changed, 31 insertions(+), 14 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index f1fbcf11c82..81fbdfaf484 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1122,16 +1122,23 @@ url = {{nouveau_url}} ; Couch Stats Resource Tracker (CSRT) [csrt] -enabled = true -;enabled = false -;should_truncate_reports = false +enable = true +enable_init_p = true +enable_reporting = true +;enable = false +;enable_init_p = false +;enable_reporting = false +; + +; Truncate reports to not include zero values for counter fields +;should_truncate_reports = true ; CSRT Rexi Server RPC Worker Spawn Tracking ; Enable these to enable additional metrics for RPC worker spawn rates ; measuring how often RPC workers are spawned by way of rexi_server:init_p. ; Mod and Function are separated by double underscores. [csrt.init_p] -enabled = false +;enable = true fabric_rpc__all_docs = true fabric_rpc__changes = true fabric_rpc__map_view = true diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index bd2df44f96c..f7468ebd3ce 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -120,7 +120,7 @@ %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer(), get_kv_node = 0 :: non_neg_integer(), get_kp_node = 0 :: non_neg_integer() - %% Add these later + %% "Example to extend CSRT" %%write_kv_node = 0 :: non_neg_integer(), %%write_kp_node = 0 :: non_neg_integer() }). @@ -143,6 +143,7 @@ | js_filtered_docs | get_kv_node | get_kp_node. + %% "Example to extend CSRT" %%| write_kv_node %%| write_kp_node. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 2551b14c7f7..ff408b7c7ea 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -473,7 +473,7 @@ t_static_map_translations(_) -> ). t_should_track_init_p(_) -> - config:set(?CSRT_INIT_P, "enabled", "true", false), + config:set(?CSRT, "enable_init_p", "true", false), Metrics = [ [fabric_rpc, all_docs, spawned], [fabric_rpc, changes, spawned], @@ -488,7 +488,7 @@ t_should_track_init_p(_) -> [?assert(should_track_init_p(M), M) || M <- Metrics]. t_should_not_track_init_p(_) -> - config:set(?CSRT_INIT_P, "enabled", "true", false), + config:set(?CSRT, "enable_init_p", "true", false), Metrics = [ [couch_db, name, spawned], [couch_db, get_db_info, spawned], diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index ff43e09febc..4e50bf83843 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -127,7 +127,7 @@ deregister_matcher(Name) -> -spec log_process_lifetime_report(PidRef :: pid_ref()) -> ok. log_process_lifetime_report(PidRef) -> - case csrt_util:is_enabled() of + case csrt_util:is_enabled() andalso csrt_util:is_enabled_reporting() of true -> maybe_report("csrt-pid-usage-lifetime", PidRef); false -> @@ -190,6 +190,7 @@ is_match(_Rctx, undefined) -> is_match(#rctx{} = Rctx, Matchers) when is_map(Matchers) -> maps:size(find_matches([Rctx], Matchers)) > 0. +%% Generate a report for the Rctx if it triggers an active Matcher -spec maybe_report(ReportName :: string(), PidRef :: maybe_pid_ref()) -> ok. maybe_report(ReportName, PidRef) -> Rctx = csrt_server:get_resource(PidRef), @@ -201,7 +202,7 @@ maybe_report(ReportName, PidRef) -> ok end. -%% Whether or not to remove zero value fields from reports +%% Whether or not to remove zero value fields from reports to save on volume -spec should_truncate_reports() -> boolean(). should_truncate_reports() -> config:get_boolean(?CSRT, "should_truncate_reports", true). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 9f3ffc1b04d..f79c7af2a97 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -14,6 +14,7 @@ -export([ is_enabled/0, + is_enabled_reporting/0, is_enabled_init_p/0, get_pid_ref/0, get_pid_ref/1, @@ -68,18 +69,19 @@ is_enabled() -> true -> rand:uniform(100) > 80; false -> - config:get_boolean(?CSRT, "enabled", true) + config:get_boolean(?CSRT, "enable", true) end. -else. -spec is_enabled() -> boolean(). is_enabled() -> %% TODO: toggle back to false before merging - config:get_boolean(?CSRT, "enabled", true). + config:get_boolean(?CSRT, "enable", true). -endif. -spec is_enabled_init_p() -> boolean(). is_enabled_init_p() -> - config:get_boolean(?CSRT_INIT_P, "enabled", true). + %% TODO: toggle back to false before merging + config:get_boolean(?CSRT, "enable_init_p", true). -spec should_track_init_p(Mod :: atom(), Func :: atom()) -> boolean(). should_track_init_p(fabric_rpc, Func) -> @@ -87,6 +89,12 @@ should_track_init_p(fabric_rpc, Func) -> should_track_init_p(_Mod, _Func) -> false. +%% Toggle to disable all reporting +-spec is_enabled_reporting() -> boolean(). +is_enabled_reporting() -> + %% TODO: toggle back to false before merging + config:get_boolean(?CSRT, "enable_reporting", true). + %% Monotnonic time now in native format using time forward only event tracking -spec tnow() -> integer(). tnow() -> @@ -463,7 +471,7 @@ enable_init_p() -> enable_init_p(base_metrics()). enable_init_p(Metrics) -> - config:set(?CSRT_INIT_P, "enabled", "true", false), + config:set(?CSRT, "enable_init_p", "true", false), enable_init_p_metrics(Metrics). enable_init_p_metrics() -> @@ -476,7 +484,7 @@ disable_init_p() -> disable_init_p(base_metrics()). disable_init_p(Metrics) -> - config:set(?CSRT_INIT_P, "enabled", "false", false), + config:set(?CSRT, "enable_init_p", "false", false), disable_init_p_metrics(Metrics). disable_init_p_metrics() -> From c41ac4fdf7eccb394f65db857eadeb5b9cd69033 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 11 Jun 2025 19:14:14 -0700 Subject: [PATCH 32/54] Rework updated_at logic --- .../src/couch_stats_resource_tracker.hrl | 3 +- src/couch_stats/src/csrt.erl | 4 +- src/couch_stats/src/csrt_server.erl | 66 +++++++++++++++++-- src/couch_stats/src/csrt_util.erl | 36 ++++++---- .../test/eunit/csrt_server_tests.erl | 40 +++++++++++ 5 files changed, 126 insertions(+), 23 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index f7468ebd3ce..ced4b5b5bda 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -16,7 +16,7 @@ %% CSRT pdict markers -define(DELTA_TA, csrt_delta_ta). --define(DELTA_TZ, csrt_delta_tz). %% T Zed instead of T0 +-define(LAST_UPDATED, csrt_last_updated). -define(PID_REF, csrt_pid_ref). %% track local ID -define(TRACKER_PID, csrt_tracker). %% tracker pid @@ -100,6 +100,7 @@ -record(rctx, { %% Metadata started_at = csrt_util:tnow(), + %% NOTE: updated_at must be after started_at to preserve time congruity updated_at = csrt_util:tnow(), pid_ref :: maybe_pid_ref() | {'_', '_'}, nonce, diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index ff408b7c7ea..04a6f99e23c 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -153,8 +153,8 @@ create_context(Type, Nonce) -> PidRef = get_pid_ref(Rctx), set_pid_ref(PidRef), try - csrt_util:set_delta_zero(Rctx), - csrt_util:set_delta_a(Rctx), + csrt_util:put_delta_a(Rctx), + csrt_util:put_updated_at(Rctx), csrt_server:create_resource(Rctx), csrt_logger:track(Rctx), PidRef diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index 8448041b6ad..65dd157e946 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -36,7 +36,9 @@ set_context_type/2, set_context_username/2, update_counter/3, - update_counters/2 + update_counter/4, + update_counters/2, + update_counters/3 ]). -include_lib("stdlib/include/ms_transform.hrl"). @@ -154,18 +156,60 @@ is_rctx_stat_field(Field) -> get_rctx_stat_field(Field) -> maps:get(Field, ?STAT_KEYS_TO_FIELDS). +%% This provides a base set of updates to include along with any other #rctx{} +%% updates. Specifically, this provides a way to automatically track and +%% increment the #rctx.updated_at field without having to do ets:lookup to find +%% the last updated_at time, or having to do ets:update_element to set a +%% specific updated_at. We trade a pdict marker to keep inc operations as only +%% a singular ets call while sneaking in updated_at. +%% Calling csrt_util:put_updated_at/1 within this function is not the cleanest, +%% but it allows us to encapsulate the automatic updated_at inclusion into the +%% ?MODULE:update_counter(s)/3-4 arity call-through while still allowing the +%% 4-arity version to be exposed to pass an empty base updates list. Isolating +%% this logic means the final arity functions operate independently of any +%% local pdict values. +-spec make_base_counter_updates() -> [] | [{rctx_field(), integer()}]. +make_base_counter_updates() -> + case csrt_util:get_updated_at() of + undefined -> + []; + LastUpdated -> + Now = csrt_util:tnow(), + csrt_util:put_updated_at(Now), + UpdatedInc = csrt_util:make_dt(LastUpdated, Now, native), + [{#rctx.updated_at, UpdatedInc}] + end. + -spec update_counter(PidRef, Field, Count) -> non_neg_integer() when PidRef :: maybe_pid_ref(), Field :: rctx_field(), Count :: non_neg_integer(). update_counter(undefined, _Field, _Count) -> 0; -update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> +update_counter(_PidRef, _Field, 0) -> + 0; +update_counter(PidRef, Field, Count) -> + %% Only call make_base_counter_updates() if PidRef, Field, Count all valid + case is_rctx_stat_field(Field) of + true -> + update_counter(PidRef, Field, Count, make_base_counter_updates()); + false -> + 0 + end. + +-spec update_counter(PidRef, Field, Count, BaseUpdates) -> non_neg_integer() when + PidRef :: maybe_pid_ref(), + Field :: rctx_field(), + Count :: non_neg_integer(), + BaseUpdates :: [] | [{rctx_field(), integer()}]. +update_counter(undefined, _Field, _Count, _BaseUpdates) -> + 0; +update_counter({_Pid, _Ref} = PidRef, Field, Count, BaseUpdates) when Count >= 0 -> case is_rctx_stat_field(Field) of true -> - Update = {get_rctx_stat_field(Field), Count}, + Updates = [{get_rctx_stat_field(Field), Count} | BaseUpdates], try - ets:update_counter(?CSRT_ETS, PidRef, Update, #rctx{pid_ref = PidRef}) + ets:update_counter(?CSRT_ETS, PidRef, Updates, #rctx{pid_ref = PidRef}) catch _:_ -> 0 @@ -179,7 +223,16 @@ update_counter({_Pid, _Ref} = PidRef, Field, Count) when Count >= 0 -> Delta :: delta(). update_counters(undefined, _Delta) -> false; -update_counters({_Pid, _Ref} = PidRef, Delta) when is_map(Delta) -> +update_counters(PidRef, Delta) when is_map(Delta) -> + update_counters(PidRef, Delta, make_base_counter_updates()). + +-spec update_counters(PidRef, Delta, BaseUpdates) -> boolean() when + PidRef :: maybe_pid_ref(), + Delta :: delta(), + BaseUpdates :: [] | [{rctx_field(), integer()}]. +update_counters(undefined, _Delta, _BaseUpdates) -> + false; +update_counters({_Pid, _Ref} = PidRef, Delta, BaseUpdates) when is_map(Delta) -> Updates = maps:fold( fun(Field, Count, Acc) -> case is_rctx_stat_field(Field) of @@ -190,10 +243,11 @@ update_counters({_Pid, _Ref} = PidRef, Delta) when is_map(Delta) -> %% Another approach would be: %% lists:all(lists:map(fun is_rctx_stat_field/1, maps:keys(Delta))) %% But that's a lot of looping for not even acumulating the update. + %% Need to drop Delta.dt either way as it's not an rctx_field Acc end end, - [], + BaseUpdates, Delta ), diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index f79c7af2a97..27cef713a07 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -40,15 +40,15 @@ extract_delta/1, get_delta/1, get_delta_a/0, - get_delta_zero/0, + get_updated_at/0, maybe_add_delta/1, maybe_add_delta/2, make_delta/1, make_dt/2, make_dt/3, rctx_delta/2, - set_delta_a/1, - set_delta_zero/1 + put_delta_a/1, + put_updated_at/1 ]). %% Extra niceties and testing facilities @@ -135,7 +135,13 @@ make_dt(A, A, _Unit) when is_integer(A) -> make_dt(A, B, Unit) when is_integer(A) andalso is_integer(B) andalso B > A -> A1 = erlang:convert_time_unit(A, native, Unit), B1 = erlang:convert_time_unit(B, native, Unit), - B1 - A1. + case B1 - A1 of + Delta when Delta > 0 -> + Delta; + _ -> + 1 + end. + %% %% Conversion API for outputting JSON @@ -340,7 +346,7 @@ make_delta(PidRef) -> TA = get_delta_a(), TB = csrt_server:get_resource(PidRef), Delta = rctx_delta(TA, TB), - set_delta_a(TB), + put_delta_a(TB), Delta. -spec rctx_delta(TA :: Rctx, TB :: Rctx) -> map(). @@ -369,17 +375,19 @@ rctx_delta(_, _) -> get_delta_a() -> erlang:get(?DELTA_TA). --spec get_delta_zero() -> maybe_rctx(). -get_delta_zero() -> - erlang:get(?DELTA_TZ). - --spec set_delta_a(TA :: rctx()) -> maybe_rctx(). -set_delta_a(TA) -> +-spec put_delta_a(TA :: rctx()) -> maybe_rctx(). +put_delta_a(TA) -> erlang:put(?DELTA_TA, TA). --spec set_delta_zero(TZ :: rctx()) -> maybe_rctx(). -set_delta_zero(TZ) -> - erlang:put(?DELTA_TZ, TZ). +-spec get_updated_at() -> maybe_rctx(). +get_updated_at() -> + erlang:get(?LAST_UPDATED). + +-spec put_updated_at(_Rctx :: rctx()) -> maybe_rctx(). +put_updated_at(#rctx{updated_at=Updated}) -> + put_updated_at(Updated); +put_updated_at(Updated) when is_integer(Updated) -> + erlang:put(?LAST_UPDATED, Updated). -spec get_pid_ref() -> maybe_pid_ref(). get_pid_ref() -> diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index e90f0783a8b..f158bd5aa2c 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -45,6 +45,7 @@ test_funs() -> ?TDEF_FE(t_changes), ?TDEF_FE(t_changes_limit_zero), ?TDEF_FE(t_changes_filtered), + ?TDEF_FE(t_updated_at), ?TDEF_FE(t_view_query), ?TDEF_FE(t_view_query_include_docs) ]. @@ -251,6 +252,45 @@ t_get_doc({_Ctx, DbName, _View}) -> ok = nonzero_local_io_assert(Rctx, io_sum), ok = assert_teardown(PidRef). +t_updated_at({_Ctx, DbName, _View}) -> + %% Same test as t_get_doc but with a timer sleep and updated_at assertion + TimeDelay = 1234, + pdebug(dbname, DbName), + DocId = "foo_17", + Context = #{ + method => 'GET', + path => "/" ++ ?b2l(DbName) ++ "/" ++ DocId + }, + {PidRef, Nonce} = coordinator_context(Context), + Rctx0 = load_rctx(PidRef), + ok = fresh_rctx_assert(Rctx0, PidRef, Nonce), + timer:sleep(TimeDelay), + _Res = fabric:open_doc(DbName, DocId, [?ADMIN_CTX]), + Rctx = load_rctx(PidRef), + %% Get RawRctx to have pre-json-converted timestamps + RawRctx = csrt:get_resource(PidRef), + pdebug(rctx, Rctx), + ok = rctx_assert(Rctx, #{ + nonce => Nonce, + db_open => 1, + rows_read => 0, + docs_read => 1, + docs_written => 0, + pid_ref => PidRef + }), + Started = csrt_util:field(started_at, RawRctx), + Updated = csrt_util:field(updated_at, RawRctx), + ?assert( + csrt_util:make_dt(Started, Updated, millisecond) > TimeDelay, + "updated_at gets updated with an expected TimeDelay" + ), + ?assert( + csrt_util:make_dt(Started, Updated, millisecond) < 2*TimeDelay, + "updated_at gets updated in a reasonable time frame" + ), + ok = nonzero_local_io_assert(Rctx, io_sum), + ok = assert_teardown(PidRef). + t_put_doc({_Ctx, DbName, View}) -> pdebug(dbname, DbName), DocId = "bar_put_1919", From e4198ed7529cce5259644a1fb1a6a50f0c7365d0 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 11 Jun 2025 21:39:20 -0700 Subject: [PATCH 33/54] Add csrt_logger:matcher_on_long_reqs --- src/couch_stats/src/csrt_logger.erl | 39 ++++++++++++++++++- .../test/eunit/csrt_logger_tests.erl | 27 +++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 4e50bf83843..e5eda272e89 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -65,6 +65,7 @@ matcher_on_worker_changes_processed/1, matcher_on_ioq_calls/1, matcher_on_nonce/1, + matcher_on_long_reqs/1, register_matcher/2, reload_matchers/0 ]). @@ -364,6 +365,42 @@ matcher_on_worker_changes_processed(Threshold) when end ). +%% Matcher on requests taking longer than Threshold milliseconds +-spec matcher_on_long_reqs(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_long_reqs(Threshold) when + is_integer(Threshold) andalso Threshold > 0 +-> + %% Threshold is in milliseconds, but we track erlang:monotonic_time/0 + %% which is in native format, a machine dependent internal representation + %% but with the guarantee of monotonically increasing. Note that this number + %% is integer(), _not_ positive_integer(), so we must use abs/2 to get a + %% positive time delta; notably, abs/2 is valid match_spec guard. + %% + %% Time warps and is relative and is complicated, so here's an example of + %% converting 10000 milliseconds into a native time format and back, then + %% using csrt_util:tnow/0 to accurately measure sleeping for 10000 ms. + %% + %% (node1@127.0.0.1)5> erlang:convert_time_unit(10000, millisecond, native). + %% 10000000000 + %% (node1@127.0.0.1)6> erlang:convert_time_unit(10000000000, native, millisecond). + %% 10000 + %% (node1@127.0.0.1)7> T0 = csrt_util:tnow(), timer:sleep(10000), T1 = csrt_util:tnow(), + %% erlang:convert_time_unit(abs(T1 - T0), native, millisecond). + %% 10000 + + Unit = millisecond, + NativeThreshold = erlang:convert_time_unit(Threshold, Unit, native), + ets:fun2ms( + fun( + #rctx{ + started_at = Started, + updated_at = Updated + } = R + ) when abs(Updated - Started) >= NativeThreshold -> + R + end + ). + -spec matcher_on_ioq_calls(Threshold :: pos_integer()) -> ets:match_spec(). matcher_on_ioq_calls(Threshold) when is_integer(Threshold) andalso Threshold > 0 @@ -396,7 +433,7 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> {rows_read, fun matcher_on_rows_read/1, 1000}, {docs_written, fun matcher_on_docs_written/1, 500}, %%{view_rows_read, fun matcher_on_rows_read/1, 1000}, - %%{slow_reqs, fun matcher_on_slow_reqs/1, 10000}, + {long_reqs, fun matcher_on_long_reqs/1, 60000}, %% in milliseconds {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, {ioq_calls, fun matcher_on_ioq_calls/1, 10000} ], diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index beab30f4e84..0bfb23d6f9b 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -24,6 +24,7 @@ -define(RCTX_RPC, #rpc_worker{from = {self(), make_ref()}}). -define(RCTX_COORDINATOR, #coordinator{method = 'GET', path = "/foo/_all_docs"}). +%% Use different values than default configs to ensure they're picked up -define(THRESHOLD_DBNAME, <<"foo">>). -define(THRESHOLD_DBNAME_IO, 91). -define(THRESHOLD_DOCS_READ, 123). @@ -31,6 +32,7 @@ -define(THRESHOLD_IOQ_CALLS, 439). -define(THRESHOLD_ROWS_READ, 143). -define(THRESHOLD_CHANGES, 79). +-define(THRESHOLD_LONG_REQS, 432). csrt_logger_reporting_works_test_() -> { @@ -56,6 +58,7 @@ csrt_logger_matchers_test_() -> ?TDEF_FE(t_matcher_on_docs_written), ?TDEF_FE(t_matcher_on_rows_read), ?TDEF_FE(t_matcher_on_worker_changes_processed), + ?TDEF_FE(t_matcher_on_long_reqs), ?TDEF_FE(t_matcher_on_ioq_calls), ?TDEF_FE(t_matcher_on_nonce), ?TDEF_FE(t_matcher_register_deregister) @@ -112,6 +115,9 @@ setup() -> integer_to_list(?THRESHOLD_CHANGES), false ), + ok = config:set( + "csrt_logger.matchers_threshold", "long_reqs", integer_to_list(?THRESHOLD_LONG_REQS), false + ), ok = config:set("csrt_logger.dbnames_io", "foo", integer_to_list(?THRESHOLD_DBNAME_IO), false), ok = config:set("csrt_logger.dbnames_io", "bar", integer_to_list(?THRESHOLD_DBNAME_IO), false), ok = config:set( @@ -290,6 +296,27 @@ t_matcher_on_worker_changes_processed(#{rctxs := Rctxs0}) -> "Changes processed matcher" ). +t_matcher_on_long_reqs(#{rctxs := Rctxs0}) -> + %% Threshold is in milliseconds, convert to native time format + Threshold = ?THRESHOLD_LONG_REQS, + NativeThreshold = erlang:convert_time_unit(Threshold, millisecond, native), + %% Native is a small timescale, make sure we have enough for a millisecond + %% measureable time delta + %% Make sure we have at least one match + Now = csrt_util:tnow(), + UpdatedAt = Now - round(NativeThreshold * 1.23), + Rctxs = [rctx_gen(#{started_at => Now, updated_at => UpdatedAt}) | Rctxs0], + DurationFilter = fun(R) -> + Started = csrt_util:field(started_at, R), + Updated = csrt_util:field(updated_at, R), + abs(Updated - Started) >= NativeThreshold + end, + ?assertEqual( + lists:sort(lists:filter(DurationFilter, Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("long_reqs"), Rctxs)), + "Long requests matcher" + ). + t_matcher_on_ioq_calls(#{rctxs := Rctxs0}) -> Threshold = ?THRESHOLD_IOQ_CALLS, %% Make sure we have at least one match From 720649d6476b50dd65056a55cf34c4dcffa2c4d5 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 11 Jun 2025 22:36:10 -0700 Subject: [PATCH 34/54] Cleanup Dialyzer findings --- .../src/couch_stats_resource_tracker.hrl | 10 ++++++++++ src/couch_stats/src/csrt.erl | 2 +- src/couch_stats/src/csrt_server.erl | 2 +- src/couch_stats/src/csrt_util.erl | 18 +++++++----------- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index ced4b5b5bda..eab303fcb70 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -163,6 +163,7 @@ -type delta() :: map(). -type maybe_delta() :: delta() | undefined. -type tagged_delta() :: {delta, maybe_delta()}. +-type term_delta() :: term() | {term(), tagged_delta()}. -type matcher_name() :: string(). -type matcher() :: {ets:match_spec(), ets:comp_match_spec()}. @@ -170,3 +171,12 @@ -type matcher_matches() :: #{matcher_name() => rctxs()} | #{}. -type maybe_matcher() :: matcher() | undefined. -type maybe_matchers() :: matchers() | undefined. + +-type maybe_integer() :: integer() | undefined. +%% This is a little awkward to type, it's a list of ets:update_counter UpdateOp's +%% where ets types the updates as `UpdateOp = {Pos, Incr}`. We can do better than +%% that because we know `Pos` is the #rctx record field index, a non_neg_integer(), +%% and similarly, we know Incr is from `csrt_util:make_dt`, which is returns at +%% least one. Ideally, we'd specify the `Pos` type sufficiently to be one of the +%% valid #rctx record field names, however, a clean solution is not obvious. +-type counter_updates_list() :: [{non_neg_integer(), pos_integer()}] | []. diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 04a6f99e23c..a07976aed3d 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -160,7 +160,7 @@ create_context(Type, Nonce) -> PidRef catch _:_ -> - csrt_server:destroy_resource(Rctx), + csrt_server:destroy_resource(PidRef), %% destroy_context(PidRef) clears the tracker too destroy_context(PidRef), false diff --git a/src/couch_stats/src/csrt_server.erl b/src/couch_stats/src/csrt_server.erl index 65dd157e946..f16dcdb2daf 100644 --- a/src/couch_stats/src/csrt_server.erl +++ b/src/couch_stats/src/csrt_server.erl @@ -168,7 +168,7 @@ get_rctx_stat_field(Field) -> %% 4-arity version to be exposed to pass an empty base updates list. Isolating %% this logic means the final arity functions operate independently of any %% local pdict values. --spec make_base_counter_updates() -> [] | [{rctx_field(), integer()}]. +-spec make_base_counter_updates() -> counter_updates_list(). make_base_counter_updates() -> case csrt_util:get_updated_at() of undefined -> diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 27cef713a07..19e005d3f0e 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -291,7 +291,7 @@ field(changes_returned, #rctx{changes_returned = Val}) -> field(ioq_calls, #rctx{ioq_calls = Val}) -> Val. --spec add_delta(T :: term(), Delta :: maybe_delta()) -> term(). +-spec add_delta(T :: term(), Delta :: tagged_delta()) -> term_delta(). add_delta(T, {delta, undefined}) -> T; add_delta(T, {delta, _} = Delta) -> @@ -299,7 +299,7 @@ add_delta(T, {delta, _} = Delta) -> add_delta(T, _Delta) -> T. --spec extract_delta(T :: term()) -> {term(), maybe_delta()}. +-spec extract_delta(T :: term_delta()) -> {term(), maybe_delta()}. extract_delta({Msg, {delta, Delta}}) -> {Msg, Delta}; extract_delta(Msg) -> @@ -309,7 +309,7 @@ extract_delta(Msg) -> get_delta(PidRef) -> {delta, make_delta(PidRef)}. --spec maybe_add_delta(T :: term()) -> term(). +-spec maybe_add_delta(T :: term()) -> term_delta(). maybe_add_delta(T) -> case is_enabled() of false -> @@ -320,7 +320,7 @@ maybe_add_delta(T) -> %% Allow for externally provided Delta in error handling scenarios %% eg in cases like rexi_server:notify_caller/3 --spec maybe_add_delta(T :: term(), Delta :: maybe_delta()) -> term(). +-spec maybe_add_delta(T :: term(), Delta :: tagged_delta()) -> term_delta(). maybe_add_delta(T, Delta) -> case is_enabled() of false -> @@ -329,13 +329,9 @@ maybe_add_delta(T, Delta) -> maybe_add_delta_int(T, Delta) end. --spec maybe_add_delta_int(T :: term(), Delta :: maybe_delta()) -> term(). -maybe_add_delta_int(T, undefined) -> - T; +-spec maybe_add_delta_int(T :: term(), Delta :: tagged_delta()) -> term_delta(). maybe_add_delta_int(T, {delta, undefined}) -> T; -maybe_add_delta_int(T, Delta) when is_map(Delta) -> - maybe_add_delta_int(T, {delta, Delta}); maybe_add_delta_int(T, {delta, _} = Delta) -> add_delta(T, Delta). @@ -379,11 +375,11 @@ get_delta_a() -> put_delta_a(TA) -> erlang:put(?DELTA_TA, TA). --spec get_updated_at() -> maybe_rctx(). +-spec get_updated_at() -> maybe_integer(). get_updated_at() -> erlang:get(?LAST_UPDATED). --spec put_updated_at(_Rctx :: rctx()) -> maybe_rctx(). +-spec put_updated_at(Updated :: rctx() | integer()) -> maybe_integer(). put_updated_at(#rctx{updated_at=Updated}) -> put_updated_at(Updated); put_updated_at(Updated) when is_integer(Updated) -> From 0a6c5569c6257614be1d8b09e958f35fcdf22c05 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Thu, 12 Jun 2025 16:55:30 -0700 Subject: [PATCH 35/54] make erlfmt-format --- src/couch_stats/src/csrt_logger.erl | 3 ++- src/couch_stats/src/csrt_util.erl | 3 +-- src/couch_stats/test/eunit/csrt_server_tests.erl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index e5eda272e89..10350d2e207 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -433,7 +433,8 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> {rows_read, fun matcher_on_rows_read/1, 1000}, {docs_written, fun matcher_on_docs_written/1, 500}, %%{view_rows_read, fun matcher_on_rows_read/1, 1000}, - {long_reqs, fun matcher_on_long_reqs/1, 60000}, %% in milliseconds + %% long_reqs Threshold in milliseconds + {long_reqs, fun matcher_on_long_reqs/1, 60000}, {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, {ioq_calls, fun matcher_on_ioq_calls/1, 10000} ], diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 19e005d3f0e..0b7e4ecb274 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -142,7 +142,6 @@ make_dt(A, B, Unit) when is_integer(A) andalso is_integer(B) andalso B > A -> 1 end. - %% %% Conversion API for outputting JSON %% @@ -380,7 +379,7 @@ get_updated_at() -> erlang:get(?LAST_UPDATED). -spec put_updated_at(Updated :: rctx() | integer()) -> maybe_integer(). -put_updated_at(#rctx{updated_at=Updated}) -> +put_updated_at(#rctx{updated_at = Updated}) -> put_updated_at(Updated); put_updated_at(Updated) when is_integer(Updated) -> erlang:put(?LAST_UPDATED, Updated). diff --git a/src/couch_stats/test/eunit/csrt_server_tests.erl b/src/couch_stats/test/eunit/csrt_server_tests.erl index f158bd5aa2c..c2241a5e606 100644 --- a/src/couch_stats/test/eunit/csrt_server_tests.erl +++ b/src/couch_stats/test/eunit/csrt_server_tests.erl @@ -285,7 +285,7 @@ t_updated_at({_Ctx, DbName, _View}) -> "updated_at gets updated with an expected TimeDelay" ), ?assert( - csrt_util:make_dt(Started, Updated, millisecond) < 2*TimeDelay, + csrt_util:make_dt(Started, Updated, millisecond) < 2 * TimeDelay, "updated_at gets updated in a reasonable time frame" ), ok = nonzero_local_io_assert(Rctx, io_sum), From 7b96f89b17ba0649bb8fe53842c007adf55e4f6b Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 13 Jun 2025 19:43:54 -0700 Subject: [PATCH 36/54] Test csrt_util:field for all #rctx{} fields --- src/couch_stats/src/csrt_util.erl | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 0b7e4ecb274..0eb1cd48a67 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -56,7 +56,8 @@ set_fabric_init_p/2, set_fabric_init_p/3, map_to_rctx/1, - field/2 + field/2, + rctx_record_info/0 ]). -include_lib("couch_stats_resource_tracker.hrl"). @@ -416,6 +417,13 @@ fabric_conf_key(Key) -> %% Double underscore to separate Mod and Func "fabric_rpc__" ++ atom_to_list(Key). +-spec rctx_record_info() -> #{fields => [rctx_field()], size => pos_integer()}. +rctx_record_info() -> + #{ + fields => record_info(fields, rctx), + size => record_info(size, rctx) + }. + -ifdef(TEST). -include_lib("couch/include/couch_eunit.hrl"). @@ -430,7 +438,8 @@ couch_stats_resource_tracker_test_() -> ?TDEF_FE(t_should_not_track_init_p_empty), ?TDEF_FE(t_should_not_track_init_p_empty_and_disabled), ?TDEF_FE(t_should_not_track_init_p_disabled), - ?TDEF_FE(t_should_not_track_init_p) + ?TDEF_FE(t_should_not_track_init_p), + ?TDEF_FE(t_should_extract_fields_properly) ] }. @@ -470,6 +479,20 @@ t_should_not_track_init_p(_) -> ], [?assert(should_track_init_p(M, F) =:= false, {M, F}) || [M, F] <- Metrics]. +t_should_extract_fields_properly(_) -> + Rctx = #rctx{}, + #{fields := Fields} = rctx_record_info(), + %% field/2 throws on invalid fields, assert that the function succeeded + TestField = fun(Field) -> + try + field(Field, Rctx), + true + catch + _:_ -> false + end + end, + [?assert(TestField(Field)) || Field <- Fields]. + enable_init_p() -> enable_init_p(base_metrics()). From 3e4736ab6800d1792b18298fa0aca082c0bec9f5 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 13 Jun 2025 20:59:53 -0700 Subject: [PATCH 37/54] Use dedicated transient CSRT supervisor --- src/couch_stats/src/couch_stats_csrt_sup.erl | 34 ++++++++++++++++++++ src/couch_stats/src/couch_stats_sup.erl | 14 ++++++-- 2 files changed, 45 insertions(+), 3 deletions(-) create mode 100644 src/couch_stats/src/couch_stats_csrt_sup.erl diff --git a/src/couch_stats/src/couch_stats_csrt_sup.erl b/src/couch_stats/src/couch_stats_csrt_sup.erl new file mode 100644 index 00000000000..bfb321b39ba --- /dev/null +++ b/src/couch_stats/src/couch_stats_csrt_sup.erl @@ -0,0 +1,34 @@ +% Licensed under the Apache License, Version 2.0 (the "License"); you may not +% use this file except in compliance with the License. You may obtain a copy of +% the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +% WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +% License for the specific language governing permissions and limitations under +% the License. + +-module(couch_stats_csrt_sup). + +-behaviour(supervisor). + +-export([ + start_link/0, + init/1 +]). + +-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). + +start_link() -> + supervisor:start_link({local, ?MODULE}, ?MODULE, []). + +init([]) -> + {ok, + { + {one_for_one, 5, 10}, [ + ?CHILD(csrt_server, worker), + ?CHILD(csrt_logger, worker) + ] + }}. diff --git a/src/couch_stats/src/couch_stats_sup.erl b/src/couch_stats/src/couch_stats_sup.erl index 826dfbac4fc..467079ca260 100644 --- a/src/couch_stats/src/couch_stats_sup.erl +++ b/src/couch_stats/src/couch_stats_sup.erl @@ -20,17 +20,25 @@ ]). -define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}). +-define(CSRT_SUP, couch_stats_csrt_sup). start_link() -> supervisor:start_link({local, ?MODULE}, ?MODULE, []). init([]) -> + %% Use a dedicated CSRT supervisor with restart strategy set to transient + %% so that if a CSRT failure arrises that triggers the sup rate limiter + %% thresholds, that shutdown signal will bubble up here and be ignored, + %% as the use of transient specifies that `normal` and `shutdown` signals + %% are ignored. + %% Switch this to `permanent` once CSRT is out of experimental stage. + CSRTSup = + {?CSRT_SUP, {?CSRT_SUP, start_link, []}, transient, infinity, supervisor, [?CSRT_SUP]}, {ok, { {one_for_one, 5, 10}, [ ?CHILD(couch_stats_server, worker), - ?CHILD(csrt_server, worker), - ?CHILD(csrt_logger, worker), - ?CHILD(couch_stats_process_tracker, worker) + ?CHILD(couch_stats_process_tracker, worker), + CSRTSup ] }}. From 7de1d9625513d5b0ffdcd7efe58ee5db8b529a52 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Fri, 13 Jun 2025 23:52:08 -0700 Subject: [PATCH 38/54] Cleanup delta handling and type specs --- src/couch_stats/src/csrt.erl | 6 ++++++ src/couch_stats/src/csrt_util.erl | 22 ++++++++++++++-------- src/rexi/src/rexi_server.erl | 1 + 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index a07976aed3d..2ac25f2cbb0 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -230,6 +230,7 @@ destroy_context() -> destroy_context(undefined) -> ok; destroy_context(PidRef) -> + %% Stopping the tracker clears the ets entry for PidRef on its way out csrt_logger:stop_tracker(), destroy_pid_ref(PidRef), ok. @@ -404,18 +405,23 @@ sorted_by(Key, Val, Agg) -> %% Delta API %% +-spec add_delta(T :: term(), Delta :: maybe_delta()) -> term_delta(). add_delta(T, Delta) -> csrt_util:add_delta(T, Delta). +-spec extract_delta(T :: term_delta()) -> {term(), maybe_delta()}. extract_delta(T) -> csrt_util:extract_delta(T). +-spec get_delta() -> tagged_delta(). get_delta() -> csrt_util:get_delta(get_pid_ref()). +-spec maybe_add_delta(T :: term()) -> term_delta(). maybe_add_delta(T) -> csrt_util:maybe_add_delta(T). +-spec maybe_add_delta(T :: term(), Delta :: maybe_delta()) -> term_delta(). maybe_add_delta(T, Delta) -> csrt_util:maybe_add_delta(T, Delta). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 0eb1cd48a67..79175c62c98 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -291,13 +291,17 @@ field(changes_returned, #rctx{changes_returned = Val}) -> field(ioq_calls, #rctx{ioq_calls = Val}) -> Val. --spec add_delta(T :: term(), Delta :: tagged_delta()) -> term_delta(). -add_delta(T, {delta, undefined}) -> +-spec add_delta(T :: term(), Delta :: maybe_delta()) -> term_delta(). +add_delta(T, undefined) -> T; -add_delta(T, {delta, _} = Delta) -> - {T, Delta}; -add_delta(T, _Delta) -> - T. +add_delta(T, Delta) when is_map(Delta) -> + add_delta_int(T, {delta, Delta}). + +-spec add_delta_int(T :: term(), Delta :: tagged_delta()) -> term_delta(). +add_delta_int(T, {delta, undefined}) -> + T; +add_delta_int(T, {delta, _} = Delta) -> + {T, Delta}. -spec extract_delta(T :: term_delta()) -> {term(), maybe_delta()}. extract_delta({Msg, {delta, Delta}}) -> @@ -320,7 +324,9 @@ maybe_add_delta(T) -> %% Allow for externally provided Delta in error handling scenarios %% eg in cases like rexi_server:notify_caller/3 --spec maybe_add_delta(T :: term(), Delta :: tagged_delta()) -> term_delta(). +-spec maybe_add_delta(T :: term(), Delta :: maybe_delta()) -> term_delta(). +maybe_add_delta(T, undefined) -> + T; maybe_add_delta(T, Delta) -> case is_enabled() of false -> @@ -333,7 +339,7 @@ maybe_add_delta(T, Delta) -> maybe_add_delta_int(T, {delta, undefined}) -> T; maybe_add_delta_int(T, {delta, _} = Delta) -> - add_delta(T, Delta). + add_delta_int(T, Delta). -spec make_delta(PidRef :: maybe_pid_ref()) -> maybe_delta(). make_delta(undefined) -> diff --git a/src/rexi/src/rexi_server.erl b/src/rexi/src/rexi_server.erl index 32a739a5d06..939be29ba88 100644 --- a/src/rexi/src/rexi_server.erl +++ b/src/rexi/src/rexi_server.erl @@ -147,6 +147,7 @@ init_p(From, {M, F, A}, Nonce) -> csrt:destroy_context(), ok; Class:Reason:Stack0 -> + %% Make a CSRT delta manually to funnel back to the caller Delta = csrt:make_delta(), csrt:destroy_context(), Stack = clean_stack(Stack0), From e41e4416c7dbc2030667823881805b34e5dbcd2b Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 16 Jun 2025 17:40:03 -0700 Subject: [PATCH 39/54] Fixup maybe_add_delta type restrucuring --- src/couch_stats/src/csrt_util.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 79175c62c98..126fdb46590 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -327,11 +327,12 @@ maybe_add_delta(T) -> -spec maybe_add_delta(T :: term(), Delta :: maybe_delta()) -> term_delta(). maybe_add_delta(T, undefined) -> T; -maybe_add_delta(T, Delta) -> +maybe_add_delta(T, Delta0) when is_map(Delta0) -> case is_enabled() of false -> T; true -> + Delta = {delta, Delta0}, maybe_add_delta_int(T, Delta) end. From 3074f77ba2d5f7d180126bcad7b08cb936b2ee9e Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Mon, 16 Jun 2025 18:29:28 -0700 Subject: [PATCH 40/54] make erlfmt-format --- rel/overlay/etc/default.ini | 2 -- 1 file changed, 2 deletions(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 81fbdfaf484..69010f032af 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1128,7 +1128,6 @@ enable_reporting = true ;enable = false ;enable_init_p = false ;enable_reporting = false -; ; Truncate reports to not include zero values for counter fields ;should_truncate_reports = true @@ -1138,7 +1137,6 @@ enable_reporting = true ; measuring how often RPC workers are spawned by way of rexi_server:init_p. ; Mod and Function are separated by double underscores. [csrt.init_p] -;enable = true fabric_rpc__all_docs = true fabric_rpc__changes = true fabric_rpc__map_view = true From 5919cc22b360f0d25bb3412f01398be152746654 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 17 Jun 2025 05:39:16 -0700 Subject: [PATCH 41/54] Add csrt:proc_window based on recon:proc_window --- .../src/couch_stats_resource_tracker.hrl | 9 ++-- src/couch_stats/src/csrt.erl | 47 ++++++++++++++++++- src/couch_stats/src/csrt_logger.erl | 41 ++++++++++++++++ src/couch_stats/src/csrt_util.erl | 10 ++-- 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index eab303fcb70..90062d817be 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -15,11 +15,12 @@ -define(CSRT_ETS, csrt_server). %% CSRT pdict markers --define(DELTA_TA, csrt_delta_ta). --define(LAST_UPDATED, csrt_last_updated). --define(PID_REF, csrt_pid_ref). %% track local ID --define(TRACKER_PID, csrt_tracker). %% tracker pid +-define(DELTA_TA, {csrt, delta_ta}). +-define(LAST_UPDATED, {csrt, last_updated}). +-define(PID_REF, {csrt, pid_ref}). %% track local ID +-define(TRACKER_PID, {csrt, tracker}). %% tracker pid +%% Stats fields -define(DB_OPEN_DOC, docs_read). -define(DB_OPEN, db_open). -define(COUCH_SERVER_OPEN, db_open). diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 2ac25f2cbb0..473134b084a 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -44,10 +44,12 @@ %% Public API -export([ + clear_pdict_markers/0, + do_report/2, is_enabled/0, is_enabled_init_p/0, - do_report/2, - maybe_report/2 + maybe_report/2, + to_json/1 ]). %% stats collection api @@ -90,6 +92,13 @@ sorted_by/3 ]). +%% Recon API Ports of https://github.com/ferd/recon/releases/tag/2.5.6 +-export([ + pid_ref_attrs/1, + pid_ref_matchspec/1, + proc_window/3 +]). + %% %% PidRef operations %% @@ -233,8 +242,21 @@ destroy_context(PidRef) -> %% Stopping the tracker clears the ets entry for PidRef on its way out csrt_logger:stop_tracker(), destroy_pid_ref(PidRef), + clear_pdict_markers(), ok. +-spec clear_pdict_markers() -> ok. +clear_pdict_markers() -> + ok = lists:foreach( + fun + ({{csrt,_} = K , _V}) -> + erlang:erase(K); + (_) -> + ok + end, + erlang:get() + ). + %% %% Public API %% @@ -267,6 +289,10 @@ maybe_report(ReportName, PidRef) -> do_report(ReportName, PidRef) -> csrt_logger:do_report(ReportName, get_resource(PidRef)). +-spec to_json(Rctx :: rctx()) -> map(). +to_json(Rctx) -> + csrt_util:to_json(Rctx). + %% %% Stat collection API %% @@ -389,6 +415,23 @@ group_by(Key, Val) -> group_by(Key, Val, Agg) -> csrt_query:group_by(Key, Val, Agg). +-spec pid_ref_matchspec(AttrName :: rctx_field()) -> term() | throw(any()). +pid_ref_matchspec(AttrName) -> + csrt_logger:pid_ref_matchspec(AttrName). + +-spec pid_ref_attrs(AttrName :: rctx_field()) -> term() | throw(any()). +pid_ref_attrs(AttrName) -> + csrt_logger:pid_ref_attrs(AttrName). + +%% This is a recon:proc_window/3 [1] port with the same core logic but +%% recon_lib:proc_attrs/1 replaced with csrt_logger:pid_ref_attrs/1, and +%% returning on pid_ref() rather than pid(). +%% [1] https://github.com/ferd/recon/blob/c2a76855be3a226a3148c0dfc21ce000b6186ef8/src/recon.erl#L268-L300 +-spec proc_window(AttrName, Num, Time) -> term() | throw(any()) when + AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). +proc_window(AttrName, Num, Time) -> + csrt_logger:proc_window(AttrName, Num, Time). + sorted(Map) -> csrt_query:sorted(Map). diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 10350d2e207..cfef4c25233 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -70,6 +70,13 @@ reload_matchers/0 ]). +%% Recon API Ports of https://github.com/ferd/recon/releases/tag/2.5.6 +-export([ + pid_ref_attrs/1, + pid_ref_matchspec/1, + proc_window/3 +]). + -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("couch_stats_resource_tracker.hrl"). @@ -407,6 +414,40 @@ matcher_on_ioq_calls(Threshold) when -> ets:fun2ms(fun(#rctx{ioq_calls = IOQCalls} = R) when IOQCalls >= Threshold -> R end). +%%-spec matcher_for_rctx_field(Field :: rctx_field()) -> ets:match_spec(). +%%matcher_for_rctx_field() -> +%% #{size := Size0, fields := Fields} = csrt_util:rctx_record_info(), +%% %% Subtract 1 as record_info size includes tuple record name +%% %% erlang:list_to_tuple([rctx | lists:duplicate(maps:get(size, csrt_util:rctx_record_info()), '_')]) +%% Size = Size - 1, + +pid_ref_matchspec(AttrName) -> + #{size := Size, field_idx := FieldIdx} = csrt_util:rctx_record_info(), + RctxMatch0 = list_to_tuple([rctx | lists:duplicate(Size - 1, '_')]), + RctxMatch1 = setelement(maps:get(pid_ref, FieldIdx) + 1 , RctxMatch0, '$1'), + RctxMatch = setelement(maps:get(AttrName, FieldIdx) + 1, RctxMatch1, '$2'), + MatchSpec = [{RctxMatch, [], [{{'$1', '$2'}}]}], + {MatchSpec, ets:match_spec_compile(MatchSpec)}. + +pid_ref_attrs(AttrName) -> + {MatchSpec, _CompMatch} = pid_ref_matchspec(AttrName), + %% Base fields at least an empty list, but we could add more info here. + %% The recon typespec is an improper list of the form: + %%Base = [Name | [{current_function, mfa()} | {initial_call, mfa()}]], + Base = [], + [{PidRef, Val, Base} || {PidRef, Val} <- ets:select(?CSRT_ETS, MatchSpec)]. + +%% This is a recon:proc_window/3 [1] port with the same core logic but +%% recon_lib:proc_attrs/1 replaced with pid_ref_attrs/1, and returning on +%% pid_ref() rather than pid(). +%% [1] https://github.com/ferd/recon/blob/c2a76855be3a226a3148c0dfc21ce000b6186ef8/src/recon.erl#L268-L300 +-spec proc_window(AttrName, Num, Time) -> term() | throw(any()) when + AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). +proc_window(AttrName, Num, Time) -> + Sample = fun() -> pid_ref_attrs(AttrName) end, + {First,Last} = recon_lib:sample(Time, Sample), + recon_lib:sublist_top_n_attrs(recon_lib:sliding_window(First, Last), Num). + -spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, badarg} when Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). add_matcher(Name, MSpec, Matchers) -> diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 126fdb46590..6d588e8d3dd 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -424,11 +424,15 @@ fabric_conf_key(Key) -> %% Double underscore to separate Mod and Func "fabric_rpc__" ++ atom_to_list(Key). --spec rctx_record_info() -> #{fields => [rctx_field()], size => pos_integer()}. +-spec rctx_record_info() -> #{fields => [rctx_field()], size => pos_integer(), field_idx => #{rctx_field() => pos_integer()}}. rctx_record_info() -> + Fields = record_info(fields, rctx), + Size = record_info(size, rctx), + Idx = maps:from_list(lists:zip(Fields, lists:seq(1, length(Fields)))), #{ - fields => record_info(fields, rctx), - size => record_info(size, rctx) + fields => Fields, + field_idx => Idx, + size => Size }. -ifdef(TEST). From 39c4675b925c7365423ee48f413ea64484a655cb Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 17 Jun 2025 05:59:26 -0700 Subject: [PATCH 42/54] Add dedicated toggle to disable #rpc_worker{} reporting --- rel/overlay/etc/default.ini | 1 + src/couch_stats/src/csrt_logger.erl | 8 +++++++- src/couch_stats/src/csrt_util.erl | 9 +++++++++ src/couch_stats/test/eunit/csrt_logger_tests.erl | 2 ++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/rel/overlay/etc/default.ini b/rel/overlay/etc/default.ini index 69010f032af..d9730a42f48 100644 --- a/rel/overlay/etc/default.ini +++ b/rel/overlay/etc/default.ini @@ -1128,6 +1128,7 @@ enable_reporting = true ;enable = false ;enable_init_p = false ;enable_reporting = false +;enable_rpc_reporting = false ; Truncate reports to not include zero values for counter fields ;should_truncate_reports = true diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index cfef4c25233..32f62074d80 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -145,9 +145,15 @@ log_process_lifetime_report(PidRef) -> %% Return a subset of Matchers for each Matcher that matches on Rctxs -spec find_matches(Rctxs :: [rctx()], Matchers :: matchers()) -> matchers(). find_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> + Rctxs1 = case csrt_util:is_enabled_rpc_reporting() of + true -> + Rctxs; + false -> + [Rctx || #rctx{type=#coordinator{}}=Rctx <- Rctxs] + end, maps:filter( fun(_Name, {_MSpec, CompMSpec}) -> - (catch ets:match_spec_run(Rctxs, CompMSpec)) =/= [] + (catch ets:match_spec_run(Rctxs1, CompMSpec)) =/= [] end, Matchers ). diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 6d588e8d3dd..78c4a055621 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -15,6 +15,7 @@ -export([ is_enabled/0, is_enabled_reporting/0, + is_enabled_rpc_reporting/0, is_enabled_init_p/0, get_pid_ref/0, get_pid_ref/1, @@ -96,6 +97,14 @@ is_enabled_reporting() -> %% TODO: toggle back to false before merging config:get_boolean(?CSRT, "enable_reporting", true). +%% Toggle to disable all reporting from #rpc_worker{} types, eg only log +%% #coordinator{} types. This is a bit of a kludge that would be better served +%% by a dynamic match spec generator, but this provides a know for disabling +%% any rpc worker logs, even if they hit the normal logging Threshold's. +-spec is_enabled_rpc_reporting() -> boolean(). +is_enabled_rpc_reporting() -> + config:get_boolean(?CSRT, "enable_rpc_reporting", false). + %% Monotnonic time now in native format using time forward only event tracking -spec tnow() -> integer(). tnow() -> diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 0bfb23d6f9b..003463edb0f 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -79,6 +79,8 @@ make_docs(Count) -> setup() -> Ctx = test_util:start_couch([fabric, couch_stats]), config:set_boolean(?CSRT, "randomize_testing", false, false), + config:set_boolean(?CSRT, "enable_reporting", true, false), + config:set_boolean(?CSRT, "enable_rpc_reporting", true, false), ok = meck:new(ioq, [passthrough]), ok = meck:expect(ioq, bypass, fun(_, _) -> false end), DbName = ?tempdb(), From c489a61caee4e558a80f540e23e9ae7387808591 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 17 Jun 2025 06:01:33 -0700 Subject: [PATCH 43/54] make erlfmt-format --- src/couch_stats/src/csrt.erl | 4 ++-- src/couch_stats/src/csrt_logger.erl | 19 ++++++++++--------- src/couch_stats/src/csrt_util.erl | 7 ++++++- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/couch_stats/src/csrt.erl b/src/couch_stats/src/csrt.erl index 473134b084a..603e444e98e 100644 --- a/src/couch_stats/src/csrt.erl +++ b/src/couch_stats/src/csrt.erl @@ -249,7 +249,7 @@ destroy_context(PidRef) -> clear_pdict_markers() -> ok = lists:foreach( fun - ({{csrt,_} = K , _V}) -> + ({{csrt, _} = K, _V}) -> erlang:erase(K); (_) -> ok @@ -428,7 +428,7 @@ pid_ref_attrs(AttrName) -> %% returning on pid_ref() rather than pid(). %% [1] https://github.com/ferd/recon/blob/c2a76855be3a226a3148c0dfc21ce000b6186ef8/src/recon.erl#L268-L300 -spec proc_window(AttrName, Num, Time) -> term() | throw(any()) when - AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). + AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). proc_window(AttrName, Num, Time) -> csrt_logger:proc_window(AttrName, Num, Time). diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 32f62074d80..3174cc827ad 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -145,12 +145,13 @@ log_process_lifetime_report(PidRef) -> %% Return a subset of Matchers for each Matcher that matches on Rctxs -spec find_matches(Rctxs :: [rctx()], Matchers :: matchers()) -> matchers(). find_matches(Rctxs, Matchers) when is_list(Rctxs) andalso is_map(Matchers) -> - Rctxs1 = case csrt_util:is_enabled_rpc_reporting() of - true -> - Rctxs; - false -> - [Rctx || #rctx{type=#coordinator{}}=Rctx <- Rctxs] - end, + Rctxs1 = + case csrt_util:is_enabled_rpc_reporting() of + true -> + Rctxs; + false -> + [Rctx || #rctx{type = #coordinator{}} = Rctx <- Rctxs] + end, maps:filter( fun(_Name, {_MSpec, CompMSpec}) -> (catch ets:match_spec_run(Rctxs1, CompMSpec)) =/= [] @@ -430,7 +431,7 @@ matcher_on_ioq_calls(Threshold) when pid_ref_matchspec(AttrName) -> #{size := Size, field_idx := FieldIdx} = csrt_util:rctx_record_info(), RctxMatch0 = list_to_tuple([rctx | lists:duplicate(Size - 1, '_')]), - RctxMatch1 = setelement(maps:get(pid_ref, FieldIdx) + 1 , RctxMatch0, '$1'), + RctxMatch1 = setelement(maps:get(pid_ref, FieldIdx) + 1, RctxMatch0, '$1'), RctxMatch = setelement(maps:get(AttrName, FieldIdx) + 1, RctxMatch1, '$2'), MatchSpec = [{RctxMatch, [], [{{'$1', '$2'}}]}], {MatchSpec, ets:match_spec_compile(MatchSpec)}. @@ -448,10 +449,10 @@ pid_ref_attrs(AttrName) -> %% pid_ref() rather than pid(). %% [1] https://github.com/ferd/recon/blob/c2a76855be3a226a3148c0dfc21ce000b6186ef8/src/recon.erl#L268-L300 -spec proc_window(AttrName, Num, Time) -> term() | throw(any()) when - AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). + AttrName :: rctx_field(), Num :: non_neg_integer(), Time :: pos_integer(). proc_window(AttrName, Num, Time) -> Sample = fun() -> pid_ref_attrs(AttrName) end, - {First,Last} = recon_lib:sample(Time, Sample), + {First, Last} = recon_lib:sample(Time, Sample), recon_lib:sublist_top_n_attrs(recon_lib:sliding_window(First, Last), Num). -spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, badarg} when diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 78c4a055621..ab50190b368 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -433,7 +433,12 @@ fabric_conf_key(Key) -> %% Double underscore to separate Mod and Func "fabric_rpc__" ++ atom_to_list(Key). --spec rctx_record_info() -> #{fields => [rctx_field()], size => pos_integer(), field_idx => #{rctx_field() => pos_integer()}}. +-spec rctx_record_info() -> + #{ + fields => [rctx_field()], + size => pos_integer(), + field_idx => #{rctx_field() => pos_integer()} + }. rctx_record_info() -> Fields = record_info(fields, rctx), Size = record_info(size, rctx), From 9735ca727aab80d74d49fc7b58a2ad870a8ec48c Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 17 Jun 2025 06:09:21 -0700 Subject: [PATCH 44/54] Remove extraneous function head --- src/couch_stats/src/csrt_util.erl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index ab50190b368..bd090a888c2 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -307,8 +307,6 @@ add_delta(T, Delta) when is_map(Delta) -> add_delta_int(T, {delta, Delta}). -spec add_delta_int(T :: term(), Delta :: tagged_delta()) -> term_delta(). -add_delta_int(T, {delta, undefined}) -> - T; add_delta_int(T, {delta, _} = Delta) -> {T, Delta}. From 6224242f415391b47899480375c2ee2ae724212e Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Tue, 17 Jun 2025 14:49:43 -0700 Subject: [PATCH 45/54] Cleanup instantiation of base #rctx{} match spec --- src/couch_stats/src/csrt_logger.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 3174cc827ad..146135c5ca1 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -429,8 +429,8 @@ matcher_on_ioq_calls(Threshold) when %% Size = Size - 1, pid_ref_matchspec(AttrName) -> - #{size := Size, field_idx := FieldIdx} = csrt_util:rctx_record_info(), - RctxMatch0 = list_to_tuple([rctx | lists:duplicate(Size - 1, '_')]), + #{field_idx := FieldIdx} = csrt_util:rctx_record_info(), + RctxMatch0 = #rctx{_ = '_'}, RctxMatch1 = setelement(maps:get(pid_ref, FieldIdx) + 1, RctxMatch0, '$1'), RctxMatch = setelement(maps:get(AttrName, FieldIdx) + 1, RctxMatch1, '$2'), MatchSpec = [{RctxMatch, [], [{{'$1', '$2'}}]}], From 97329a7e285ed189cd28cb3e3980ac9a67a0b7fb Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 16:38:50 -0700 Subject: [PATCH 46/54] Fix csrt_logger dbname io tests --- src/couch_stats/src/csrt_logger.erl | 2 +- src/couch_stats/test/eunit/csrt_logger_tests.erl | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 146135c5ca1..222815dcb0a 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -334,7 +334,7 @@ matcher_on_dbname_io_threshold(DbName, Threshold) when } = R ) when DbName =:= DbName1 andalso - ((IOQ > Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or + ((IOQ >= Threshold) or (KVN >= Threshold) or (KPN >= Threshold) or (Docs >= Threshold) or (Rows >= Threshold)) -> R diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 003463edb0f..6055a9980a5 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -40,6 +40,7 @@ csrt_logger_reporting_works_test_() -> fun setup_reporting/0, fun teardown_reporting/1, [ + ?TDEF_FE(t_enablement), ?TDEF_FE(t_do_report), ?TDEF_FE(t_do_lifetime_report), ?TDEF_FE(t_do_status_report) @@ -53,6 +54,7 @@ csrt_logger_matchers_test_() -> fun teardown/1, [ %%?TDEF_FE(t_matcher_on_dbname), %% TODO: add back in or delete + ?TDEF_FE(t_enablement), ?TDEF_FE(t_matcher_on_dbnames_io), ?TDEF_FE(t_matcher_on_docs_read), ?TDEF_FE(t_matcher_on_docs_written), @@ -152,7 +154,7 @@ rctx_gen(Opts0) -> R = fun() -> rand:uniform(?RCTX_RANGE) end, R10 = fun() -> 3 + rand:uniform(round(?RCTX_RANGE / 10)) end, Occasional = one_of([0, 0, 0, 0, 0, R]), - Nonce = one_of(["9c54fa9283", "foobar7799", lists:duplicate(10, fun nonce/0)]), + Nonce = one_of(["9c54fa9283", "foobar7799" | lists:duplicate(10, fun nonce/0)]), Base = #{ dbname => DbnameGen, db_open => R10, @@ -210,6 +212,11 @@ jrctx(Rctx) -> JRctx end. +t_enablement(#{}) -> + ?assert(csrt_util:is_enabled(), "CSRT is enabled"), + ?assert(csrt_util:is_enabled_reporting(), "CSRT reporting is enabled"), + ?assert(csrt_util:is_enabled_rpc_reporting(), "CSRT RPC reporting is enabled"). + t_do_report(#{rctx := Rctx}) -> JRctx = jrctx(Rctx), ReportName = "foo", @@ -485,7 +492,7 @@ matcher_for_dbname_io(Dbname0, Threshold) -> DbnameA = csrt_util:field(dbname, Rctx), Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read], Vals = [{F, csrt_util:field(F, Rctx)} || F <- Fields], - Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun(V) -> V >= Threshold end, Vals) + Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun({_K,V}) -> V >= Threshold end, Vals) end. nonce() -> From 51667627f960c41085093d29595459570f06bc31 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 17:06:31 -0700 Subject: [PATCH 47/54] make erlfmt-format --- src/couch_stats/test/eunit/csrt_logger_tests.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 6055a9980a5..5268b876877 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -492,7 +492,7 @@ matcher_for_dbname_io(Dbname0, Threshold) -> DbnameA = csrt_util:field(dbname, Rctx), Fields = [ioq_calls, get_kv_node, get_kp_node, docs_read, rows_read], Vals = [{F, csrt_util:field(F, Rctx)} || F <- Fields], - Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun({_K,V}) -> V >= Threshold end, Vals) + Dbname =:= mem3:dbname(DbnameA) andalso lists:any(fun({_K, V}) -> V >= Threshold end, Vals) end. nonce() -> From 45a94e58817dee647cd70d874b90eb5fb315eb28 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 18:21:49 -0700 Subject: [PATCH 48/54] Cleanup matchers --- src/couch_stats/src/csrt_logger.erl | 10 ++++------ .../test/eunit/csrt_logger_tests.erl | 18 ++++-------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 222815dcb0a..0fdde9f3838 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -62,7 +62,7 @@ matcher_on_docs_read/1, matcher_on_docs_written/1, matcher_on_rows_read/1, - matcher_on_worker_changes_processed/1, + matcher_on_changes_processed/1, matcher_on_ioq_calls/1, matcher_on_nonce/1, matcher_on_long_reqs/1, @@ -364,8 +364,8 @@ matcher_on_rows_read(Threshold) when matcher_on_nonce(Nonce) -> ets:fun2ms(fun(#rctx{nonce = Nonce1} = R) when Nonce =:= Nonce1 -> R end). --spec matcher_on_worker_changes_processed(Threshold :: pos_integer()) -> ets:match_spec(). -matcher_on_worker_changes_processed(Threshold) when +-spec matcher_on_changes_processed(Threshold :: pos_integer()) -> ets:match_spec(). +matcher_on_changes_processed(Threshold) when is_integer(Threshold) andalso Threshold > 0 -> ets:fun2ms( @@ -477,13 +477,11 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> %% Standard matchers to conditionally enable DefaultMatchers = [ {docs_read, fun matcher_on_docs_read/1, 1000}, - %%{dbname, fun matcher_on_dbname/1, <<"foo">>}, {rows_read, fun matcher_on_rows_read/1, 1000}, {docs_written, fun matcher_on_docs_written/1, 500}, - %%{view_rows_read, fun matcher_on_rows_read/1, 1000}, %% long_reqs Threshold in milliseconds {long_reqs, fun matcher_on_long_reqs/1, 60000}, - {worker_changes_processed, fun matcher_on_worker_changes_processed/1, 1000}, + {changes_processed, fun matcher_on_changes_processed/1, 1000}, {ioq_calls, fun matcher_on_ioq_calls/1, 10000} ], diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 5268b876877..18f4ee31981 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -53,13 +53,12 @@ csrt_logger_matchers_test_() -> fun setup/0, fun teardown/1, [ - %%?TDEF_FE(t_matcher_on_dbname), %% TODO: add back in or delete ?TDEF_FE(t_enablement), ?TDEF_FE(t_matcher_on_dbnames_io), ?TDEF_FE(t_matcher_on_docs_read), ?TDEF_FE(t_matcher_on_docs_written), ?TDEF_FE(t_matcher_on_rows_read), - ?TDEF_FE(t_matcher_on_worker_changes_processed), + ?TDEF_FE(t_matcher_on_changes_processed), ?TDEF_FE(t_matcher_on_long_reqs), ?TDEF_FE(t_matcher_on_ioq_calls), ?TDEF_FE(t_matcher_on_nonce), @@ -115,7 +114,7 @@ setup() -> ), ok = config:set( "csrt_logger.matchers_threshold", - "worker_changes_processed", + "changes_processed", integer_to_list(?THRESHOLD_CHANGES), false ), @@ -251,15 +250,6 @@ t_do_status_report(#{rctx := Rctx}) -> "CSRT couch_log:report" ). -t_matcher_on_dbname(#{rctx := _Rctx, rctxs := Rctxs0}) -> - %% Make sure we have at least one match - Rctxs = [rctx_gen(#{dbname => <<"foo">>}) | Rctxs0], - ?assertEqual( - lists:sort(lists:filter(matcher_on(dbname, <<"foo">>), Rctxs)), - lists:sort(lists:filter(matcher_for_csrt("dbname"), Rctxs)), - "Dbname matcher on <<\"foo\">>" - ). - t_matcher_on_docs_read(#{rctxs := Rctxs0}) -> Threshold = ?THRESHOLD_DOCS_READ, %% Make sure we have at least one match @@ -290,7 +280,7 @@ t_matcher_on_rows_read(#{rctxs := Rctxs0}) -> "Rows read matcher" ). -t_matcher_on_worker_changes_processed(#{rctxs := Rctxs0}) -> +t_matcher_on_changes_processed(#{rctxs := Rctxs0}) -> Threshold = ?THRESHOLD_CHANGES, %% Make sure we have at least one match Rctxs = [rctx_gen(#{rows_read => Threshold + 10}) | Rctxs0], @@ -301,7 +291,7 @@ t_matcher_on_worker_changes_processed(#{rctxs := Rctxs0}) -> end, ?assertEqual( lists:sort(lists:filter(ChangesFilter, Rctxs)), - lists:sort(lists:filter(matcher_for_csrt("worker_changes_processed"), Rctxs)), + lists:sort(lists:filter(matcher_for_csrt("changes_processed"), Rctxs)), "Changes processed matcher" ). From 24d56262d82d9bbe4120a1f99bc139c3d3d9b5f3 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 19:31:47 -0700 Subject: [PATCH 49/54] Rework csrt_logger:add_matcher error type --- .../src/couch_stats_resource_tracker.hrl | 6 ++++ src/couch_stats/src/csrt_logger.erl | 30 ++++++++----------- .../test/eunit/csrt_logger_tests.erl | 10 ++++++- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index 90062d817be..b47edc57575 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -36,6 +36,12 @@ -define(ROWS_READ, rows_read). -define(FRPC_CHANGES_RETURNED, changes_returned). +%% csrt_logger matcher keys +-define(MATCHERS_KEY, {?MODULE, all_csrt_matchers}). +-define(CONF_MATCHERS_ENABLED, "csrt_logger.matchers_enabled"). +-define(CONF_MATCHERS_THRESHOLD, "csrt_logger.matchers_threshold"). +-define(CONF_MATCHERS_DBNAMES, "csrt_logger.dbnames_io"). + %% Mapping of couch_stat metric names to #rctx{} field names. %% These are used for fields that we inc a counter on. -define(STATS_TO_KEYS, #{ diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 0fdde9f3838..f5deaf41d3c 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -80,11 +80,6 @@ -include_lib("stdlib/include/ms_transform.hrl"). -include_lib("couch_stats_resource_tracker.hrl"). --define(MATCHERS_KEY, {?MODULE, all_csrt_matchers}). --define(CONF_MATCHERS_ENABLED, "csrt_logger.matchers_enabled"). --define(CONF_MATCHERS_THRESHOLD, "csrt_logger.matchers_threshold"). --define(CONF_MATCHERS_DBNAMES, "csrt_logger.dbnames_io"). - -record(st, { registered_matchers = #{} }). @@ -278,7 +273,7 @@ handle_call({register, Name, MSpec}, _From, #st{registered_matchers = RMatchers} {ok, RMatchers1} -> ok = initialize_matchers(RMatchers1), {reply, ok, St#st{registered_matchers = RMatchers1}}; - {error, badarg} = Error -> + {error, {invalid_ms, _, _}} = Error -> {reply, Error, St} end; handle_call({deregister, Name}, _From, #st{registered_matchers = RMatchers} = St) -> @@ -455,7 +450,7 @@ proc_window(AttrName, Num, Time) -> {First, Last} = recon_lib:sample(Time, Sample), recon_lib:sublist_top_n_attrs(recon_lib:sliding_window(First, Last), Num). --spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, badarg} when +-spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, {invalid_ms, string(), ets:match_spec()}} when Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). add_matcher(Name, MSpec, Matchers) -> try ets:match_spec_compile(MSpec) of @@ -465,7 +460,7 @@ add_matcher(Name, MSpec, Matchers) -> {ok, Matchers1} catch error:badarg -> - {error, badarg} + {error, {invalid_ms, Name, MSpec}} end. -spec set_matchers_term(Matchers :: matchers()) -> ok. @@ -496,9 +491,9 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> case add_matcher(Name, MatchGenFunc(Threshold), Matchers0) of {ok, Matchers1} -> Matchers1; - {error, badarg} -> - couch_log:warning("[~p] Failed to initialize matcher: ~p", [ - ?MODULE, Name + {error, {invalid_ms, NameE, MSpecE}} -> + couch_log:warning("[~p] Failed to initialize matcher[~p]: ~p", [ + ?MODULE, NameE, MSpecE ]), Matchers0 end; @@ -521,9 +516,9 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> case add_matcher(Name, MSpec, Matchers0) of {ok, Matchers1} -> Matchers1; - {error, badarg} -> - couch_log:warning("[~p] Failed to initialize matcher: ~p", [ - ?MODULE, Name + {error, {invalid_ms, NameE, MSpecE}} -> + couch_log:warning("[~p] Failed to initialize matcher[~p]: ~p", [ + ?MODULE, NameE, MSpecE ]), Matchers0 end; @@ -531,9 +526,10 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> Matchers0 catch error:badarg -> - couch_log:warning("[~p] Failed to initialize dbname io matcher on: ~p", [ - ?MODULE, Dbname - ]) + couch_log:warning("[~p] Failed to initialize dbname io matcher on[~p]: ~p", [ + ?MODULE, Dbname, Value + ]), + Matchers0 end end, Matchers, diff --git a/src/couch_stats/test/eunit/csrt_logger_tests.erl b/src/couch_stats/test/eunit/csrt_logger_tests.erl index 18f4ee31981..8d781de76e3 100644 --- a/src/couch_stats/test/eunit/csrt_logger_tests.erl +++ b/src/couch_stats/test/eunit/csrt_logger_tests.erl @@ -136,7 +136,7 @@ teardown(#{ctx := Ctx, dbname := DbName}) -> setup_reporting() -> Ctx = setup(), - ok = meck:new(couch_log), + ok = meck:new(couch_log, [passthrough]), ok = meck:expect(couch_log, report, fun(_, _) -> true end), Ctx. @@ -212,6 +212,9 @@ jrctx(Rctx) -> end. t_enablement(#{}) -> + %% Set an invalid match spec to ensure csrt_logger is resilient + config:set(?CONF_MATCHERS_DBNAMES, "foobar", "lkajsdfkjkkadfjkajkf", false), + ?assertEqual(ok, csrt_logger:reload_matchers(), "reloads even with bad matcher specs set"), ?assert(csrt_util:is_enabled(), "CSRT is enabled"), ?assert(csrt_util:is_enabled_reporting(), "CSRT reporting is enabled"), ?assert(csrt_util:is_enabled_rpc_reporting(), "CSRT RPC reporting is enabled"). @@ -380,6 +383,11 @@ t_matcher_register_deregister(#{rctxs := Rctxs0}) -> Rctxs = [ExtraRctx | Rctxs0], ?assertEqual(#{}, csrt_logger:get_registered_matchers(), "no current registered matchers"), + ?assertEqual( + {error, {invalid_ms, "bad_spec", "fdsa"}}, + csrt_logger:register_matcher("bad_spec", "fdsa"), + "register bad matcher fails" + ), ?assertEqual(ok, csrt_logger:register_matcher(MName, MSpec), "register matcher"), CompMSpec = test_util:wait( fun() -> From 6533bc7ea8c0f8de0d4cfee91d217852734cf2e1 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 19:36:22 -0700 Subject: [PATCH 50/54] Cleanup Dialyzer and a few other things --- src/config/src/config_listener_mon.erl | 1 - .../src/couch_stats_resource_tracker.hrl | 38 +++++++++---------- src/couch_stats/src/csrt_logger.erl | 5 +++ src/couch_stats/src/csrt_query.erl | 8 ---- 4 files changed, 24 insertions(+), 28 deletions(-) diff --git a/src/config/src/config_listener_mon.erl b/src/config/src/config_listener_mon.erl index 898ecd183de..b661811cfde 100644 --- a/src/config/src/config_listener_mon.erl +++ b/src/config/src/config_listener_mon.erl @@ -13,7 +13,6 @@ -module(config_listener_mon). -behaviour(gen_server). --dialyzer({nowarn_function, init/1}). -export([ subscribe/2, diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index b47edc57575..e950f3ea2b8 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -106,31 +106,31 @@ -record(rctx, { %% Metadata - started_at = csrt_util:tnow(), + started_at = csrt_util:tnow() :: integer() | '_', %% NOTE: updated_at must be after started_at to preserve time congruity - updated_at = csrt_util:tnow(), - pid_ref :: maybe_pid_ref() | {'_', '_'}, - nonce, + updated_at = csrt_util:tnow() :: integer() | '_', + pid_ref :: maybe_pid_ref() | {'_', '_'} | '_', + nonce :: nonce() | undefined | '_', type :: rctx_type() | undefined | '_', - dbname, - username, + dbname :: dbname() | undefined | '_', + username :: username() | undefined | '_', %% Stats counters - db_open = 0, - docs_read = 0 :: non_neg_integer(), - docs_written = 0 :: non_neg_integer(), - rows_read = 0 :: non_neg_integer(), - changes_returned = 0 :: non_neg_integer(), - ioq_calls = 0 :: non_neg_integer(), - js_filter = 0 :: non_neg_integer(), - js_filtered_docs = 0 :: non_neg_integer(), + db_open = 0 :: non_neg_integer() | '_', + docs_read = 0 :: non_neg_integer() | '_', + docs_written = 0 :: non_neg_integer() | '_', + rows_read = 0 :: non_neg_integer() | '_', + changes_returned = 0 :: non_neg_integer() | '_', + ioq_calls = 0 :: non_neg_integer() | '_', + js_filter = 0 :: non_neg_integer() | '_', + js_filtered_docs = 0 :: non_neg_integer() | '_', %% TODO: switch record definitions to be macro based, eg: - %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer(), - get_kv_node = 0 :: non_neg_integer(), - get_kp_node = 0 :: non_neg_integer() + %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer() | '_', + get_kv_node = 0 :: non_neg_integer() | '_', + get_kp_node = 0 :: non_neg_integer() | '_' %% "Example to extend CSRT" - %%write_kv_node = 0 :: non_neg_integer(), - %%write_kp_node = 0 :: non_neg_integer() + %%write_kv_node = 0 :: non_neg_integer() | '_', + %%write_kp_node = 0 :: non_neg_integer() | '_' }). -type rctx_field() :: diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index f5deaf41d3c..5d2fe17a627 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -423,6 +423,7 @@ matcher_on_ioq_calls(Threshold) when %% %% erlang:list_to_tuple([rctx | lists:duplicate(maps:get(size, csrt_util:rctx_record_info()), '_')]) %% Size = Size - 1, +-spec pid_ref_matchspec(AttrName :: rctx_field()) -> matcher() | throw(any()). pid_ref_matchspec(AttrName) -> #{field_idx := FieldIdx} = csrt_util:rctx_record_info(), RctxMatch0 = #rctx{_ = '_'}, @@ -431,6 +432,7 @@ pid_ref_matchspec(AttrName) -> MatchSpec = [{RctxMatch, [], [{{'$1', '$2'}}]}], {MatchSpec, ets:match_spec_compile(MatchSpec)}. +-spec pid_ref_attrs(AttrName :: rctx_field()) -> list() | throw(any()). pid_ref_attrs(AttrName) -> {MatchSpec, _CompMatch} = pid_ref_matchspec(AttrName), %% Base fields at least an empty list, but we could add more info here. @@ -570,6 +572,9 @@ handle_config_change(?CONF_MATCHERS_ENABLED, _Key, _Val, _Persist, St) -> handle_config_change(?CONF_MATCHERS_THRESHOLD, _Key, _Val, _Persist, St) -> ok = gen_server:call(?MODULE, reload_matchers, infinity), {ok, St}; +handle_config_change(?CONF_MATCHERS_DBNAMES, _Key, _Val, _Persist, St) -> + ok = gen_server:call(?MODULE, reload_matchers, infinity), + {ok, St}; handle_config_change(_Sec, _Key, _Val, _Persist, St) -> {ok, St}. diff --git a/src/couch_stats/src/csrt_query.erl b/src/couch_stats/src/csrt_query.erl index a813b4c4594..5c1f7cd7ae1 100644 --- a/src/couch_stats/src/csrt_query.erl +++ b/src/couch_stats/src/csrt_query.erl @@ -118,12 +118,6 @@ count_by(KeyFun) -> group_by(KeyFun, ValFun) -> group_by(KeyFun, ValFun, fun erlang:'+'/2). -%% eg: group_by(mfa, docs_read). -%% eg: group_by(fun(#rctx{mfa=MFA,docs_read=DR}) -> {MFA, DR} end, ioq_calls). -%% eg: ^^ or: group_by([mfa, docs_read], ioq_calls). -%% eg: group_by([username, dbname, mfa], docs_read). -%% eg: group_by([username, dbname, mfa], ioq_calls). -%% eg: group_by([username, dbname, mfa], js_filters). group_by(KeyL, ValFun, AggFun) when is_list(KeyL) -> KeyFun = fun(Ele) -> list_to_tuple([field(Ele, Key) || Key <- KeyL]) end, group_by(KeyFun, ValFun, AggFun); @@ -154,8 +148,6 @@ sorted(Map) when is_map(Map) -> shortened(L) -> lists:sublist(L, 10). -%% eg: sorted_by([username, dbname, mfa], ioq_calls) -%% eg: sorted_by([dbname, mfa], doc_reads) sorted_by(KeyFun) -> shortened(sorted(count_by(KeyFun))). sorted_by(KeyFun, ValFun) -> shortened(sorted(group_by(KeyFun, ValFun))). sorted_by(KeyFun, ValFun, AggFun) -> shortened(sorted(group_by(KeyFun, ValFun, AggFun))). From 7497f09329947b44b734d3aafa8a47289e921651 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 20:03:56 -0700 Subject: [PATCH 51/54] Simple make_dt time conversions --- src/couch_stats/src/csrt_util.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index bd090a888c2..02c3587a6e4 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -143,12 +143,14 @@ make_dt(A, A, _Unit) when is_integer(A) -> %% possible positive integer value delta. 1; make_dt(A, B, Unit) when is_integer(A) andalso is_integer(B) andalso B > A -> - A1 = erlang:convert_time_unit(A, native, Unit), - B1 = erlang:convert_time_unit(B, native, Unit), - case B1 - A1 of + case erlang:convert_time_unit(abs(B - A), native, Unit) of Delta when Delta > 0 -> Delta; _ -> + %% Handle case where Delta is smaller than a whole Unit, eg: + %% Unit = millisecond, + %% (node1@127.0.0.1)2> erlang:convert_time_unit(423, native, Unit). + %% 0 1 end. From 9016bd13dfb16123754bf0d15765d26cfa0a8a50 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 20:11:02 -0700 Subject: [PATCH 52/54] Cleanup MatcherGen error handling --- src/couch_stats/src/csrt_logger.erl | 28 +++++++++++++++++----------- src/couch_stats/src/csrt_util.erl | 3 +++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index 5d2fe17a627..a6d77196fdc 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -100,7 +100,6 @@ tracker({Pid, _Ref} = PidRef) -> MonRef = erlang:monitor(process, Pid), receive stop -> - %% TODO: do we need cleanup here? log_process_lifetime_report(PidRef), csrt_server:destroy_resource(PidRef), ok; @@ -488,18 +487,25 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> Name = atom_to_list(Name0), case matcher_enabled(Name) of true -> - Threshold = matcher_threshold(Name, Threshold0), - %% TODO: handle errors from Func - case add_matcher(Name, MatchGenFunc(Threshold), Matchers0) of - {ok, Matchers1} -> - Matchers1; - {error, {invalid_ms, NameE, MSpecE}} -> - couch_log:warning("[~p] Failed to initialize matcher[~p]: ~p", [ - ?MODULE, NameE, MSpecE - ]), - Matchers0 + %% Wrap in a try-catch to handle MatcherGen errors + try + Threshold = matcher_threshold(Name, Threshold0), + case add_matcher(Name, MatchGenFunc(Threshold), Matchers0) of + {ok, Matchers1} -> + Matchers1; + {error, {invalid_ms, NameE, MSpecE}} -> + couch_log:warning("[~p] Failed to initialize matcher[~p]: ~p", [ + ?MODULE, NameE, MSpecE + ]), + Matchers0 + end + catch _:_ -> + Matchers0 end; false -> + couch_log:warning("[~p] Failed to initialize matcher: ~p", [ + ?MODULE, Name + ]), Matchers0 end end, diff --git a/src/couch_stats/src/csrt_util.erl b/src/couch_stats/src/csrt_util.erl index 02c3587a6e4..63782d4e75c 100644 --- a/src/couch_stats/src/csrt_util.erl +++ b/src/couch_stats/src/csrt_util.erl @@ -374,6 +374,9 @@ rctx_delta(#rctx{} = TA, #rctx{} = TB) -> get_kv_node => TB#rctx.get_kv_node - TA#rctx.get_kv_node, db_open => TB#rctx.db_open - TA#rctx.db_open, ioq_calls => TB#rctx.ioq_calls - TA#rctx.ioq_calls, + %% "Example to extend CSRT" + %% write_kp_node => TB#rctx.write_kp_node - TA#rctx.write_kp_node, + %% write_kv_node => TB#rctx.write_kv_node - TA#rctx.write_kv_node, dt => make_dt(TA#rctx.updated_at, TB#rctx.updated_at) }, %% TODO: reevaluate this decision From 1db2e8d8726040c8f18a1d45781bc308757c9aa5 Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 20:12:05 -0700 Subject: [PATCH 53/54] make erlfmt-format --- src/config/src/config_listener_mon.erl | 1 - src/couch_stats/src/csrt_logger.erl | 9 ++++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/config/src/config_listener_mon.erl b/src/config/src/config_listener_mon.erl index b661811cfde..b74a6130622 100644 --- a/src/config/src/config_listener_mon.erl +++ b/src/config/src/config_listener_mon.erl @@ -13,7 +13,6 @@ -module(config_listener_mon). -behaviour(gen_server). - -export([ subscribe/2, start_link/2 diff --git a/src/couch_stats/src/csrt_logger.erl b/src/couch_stats/src/csrt_logger.erl index a6d77196fdc..659524df744 100644 --- a/src/couch_stats/src/csrt_logger.erl +++ b/src/couch_stats/src/csrt_logger.erl @@ -451,7 +451,9 @@ proc_window(AttrName, Num, Time) -> {First, Last} = recon_lib:sample(Time, Sample), recon_lib:sublist_top_n_attrs(recon_lib:sliding_window(First, Last), Num). --spec add_matcher(Name, MSpec, Matchers) -> {ok, matchers()} | {error, {invalid_ms, string(), ets:match_spec()}} when +-spec add_matcher(Name, MSpec, Matchers) -> + {ok, matchers()} | {error, {invalid_ms, string(), ets:match_spec()}} +when Name :: string(), MSpec :: ets:match_spec(), Matchers :: matchers(). add_matcher(Name, MSpec, Matchers) -> try ets:match_spec_compile(MSpec) of @@ -499,8 +501,9 @@ initialize_matchers(RegisteredMatchers) when is_map(RegisteredMatchers) -> ]), Matchers0 end - catch _:_ -> - Matchers0 + catch + _:_ -> + Matchers0 end; false -> couch_log:warning("[~p] Failed to initialize matcher: ~p", [ From 73a58932f4c9a635fa1bb5eacb62dd795acf186c Mon Sep 17 00:00:00 2001 From: Russell Branca Date: Wed, 18 Jun 2025 20:16:40 -0700 Subject: [PATCH 54/54] Remove debug TODO --- src/couch_stats/src/couch_stats_resource_tracker.hrl | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/couch_stats/src/couch_stats_resource_tracker.hrl b/src/couch_stats/src/couch_stats_resource_tracker.hrl index e950f3ea2b8..aec21115e76 100644 --- a/src/couch_stats/src/couch_stats_resource_tracker.hrl +++ b/src/couch_stats/src/couch_stats_resource_tracker.hrl @@ -124,8 +124,6 @@ ioq_calls = 0 :: non_neg_integer() | '_', js_filter = 0 :: non_neg_integer() | '_', js_filtered_docs = 0 :: non_neg_integer() | '_', - %% TODO: switch record definitions to be macro based, eg: - %% ?COUCH_BT_GET_KP_NODE = 0 :: non_neg_integer() | '_', get_kv_node = 0 :: non_neg_integer() | '_', get_kp_node = 0 :: non_neg_integer() | '_' %% "Example to extend CSRT"