Skip to content

Commit c99cfd6

Browse files
authored
feat(new_metrics): support server_stat command showing some important server-level metrics (part 1) (#2085)
As the 1st part that support `server_stat` command, both built-in metrics, the usage of virtual and physical memory would be shown.
1 parent 5b9f46a commit c99cfd6

File tree

10 files changed

+364
-80
lines changed

10 files changed

+364
-80
lines changed

.clang-tidy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# https://releases.llvm.org/14.0.0/tools/clang/tools/extra/docs/clang-tidy/index.html
1919

2020
CheckOptions: []
21-
Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-modernize-use-trailing-return-type,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers'
21+
Checks: 'abseil-*,boost-*,bugprone-*,cert-*,clang-analyzer-*,concurrency-*,cppcoreguidelines-*,darwin-*,fuchsia-*,google-*,hicpp-*,linuxkernel-*,llvm-*,misc-*,modernize-*,performance-*,portability-*,readability-*,-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-misc-definitions-in-headers,-modernize-use-trailing-return-type,-cppcoreguidelines-macro-usage,-modernize-replace-disallow-copy-and-assign-macro,-bugprone-macro-parentheses,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-concurrency-mt-unsafe,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers'
2222
ExtraArgs:
2323
ExtraArgsBefore: []
2424
FormatStyle: none

.github/workflows/module_labeler_conf.yml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
# under the License.
1717
---
1818
permissions:
19-
contents: write
19+
contents: 'write'
20+
pull-requests: 'write'
2021

2122
github:
2223
- .github/**/*

build_tools/clang_tidy.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ def tidy_on_path(path):
6060
"clang-tidy",
6161
"-p0",
6262
"-path", BUILD_PATH,
63-
"-checks=-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-modernize-use-trailing-return-type,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers",
63+
"-checks=-cppcoreguidelines-pro-type-union-access,-llvm-include-order,-misc-definitions-in-headers,-modernize-use-trailing-return-type,-cppcoreguidelines-macro-usage,-modernize-replace-disallow-copy-and-assign-macro,-bugprone-macro-parentheses,-cppcoreguidelines-avoid-non-const-global-variables,-fuchsia-statically-constructed-objects,-fuchsia-overloaded-operator,-bugprone-easily-swappable-parameters,-cppcoreguidelines-non-private-member-variables-in-classes,-misc-non-private-member-variables-in-classes,-cppcoreguidelines-pro-bounds-array-to-pointer-decay,-hicpp-no-array-decay,-hicpp-named-parameter,-readability-named-parameter,-cppcoreguidelines-pro-bounds-pointer-arithmetic,-readability-function-cognitive-complexity,-cert-err58-cpp,-cppcoreguidelines-avoid-c-arrays,-hicpp-avoid-c-arrays,-modernize-avoid-c-arrays,-cppcoreguidelines-owning-memory,-cppcoreguidelines-pro-type-const-cast,-concurrency-mt-unsafe,-readability-identifier-length,-fuchsia-default-arguments-calls,-google-readability-avoid-underscore-in-googletest-name,-cppcoreguidelines-avoid-magic-numbers,-readability-magic-numbers",
6464
"-extra-arg=-language=c++",
6565
"-extra-arg=-std=c++17",
6666
"-extra-arg=-Ithirdparty/output/include"]

src/shell/command_helper.h

Lines changed: 82 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -709,30 +709,67 @@ inline std::vector<dsn::http_result> get_metrics(const std::vector<node_desc> &n
709709
return results;
710710
}
711711

712+
// Adapt the result returned by `get_metrics` into the structure that could be processed by
713+
// `remote_command`.
714+
template <typename... Args>
715+
inline dsn::error_s process_get_metrics_result(const dsn::http_result &result,
716+
const node_desc &node,
717+
const char *what,
718+
Args &&...args)
719+
{
720+
if (dsn_unlikely(!result.error())) {
721+
return FMT_ERR(result.error().code(),
722+
"ERROR: query {} metrics from node {} failed, msg={}",
723+
fmt::format(what, std::forward<Args>(args)...),
724+
node.hp,
725+
result.error());
726+
}
727+
728+
if (dsn_unlikely(result.status() != dsn::http_status_code::kOk)) {
729+
return FMT_ERR(dsn::ERR_HTTP_ERROR,
730+
"ERROR: query {} metrics from node {} failed, http_status={}, msg={}",
731+
fmt::format(what, std::forward<Args>(args)...),
732+
node.hp,
733+
dsn::get_http_status_message(result.status()),
734+
result.body());
735+
}
736+
737+
return dsn::error_s::ok();
738+
}
739+
712740
#define RETURN_SHELL_IF_GET_METRICS_FAILED(result, node, what, ...) \
713741
do { \
714-
if (dsn_unlikely(!result.error())) { \
715-
std::cout << "ERROR: send http request to query " << fmt::format(what, ##__VA_ARGS__) \
716-
<< " metrics from node " << node.hp << " failed: " << result.error() \
717-
<< std::endl; \
718-
return true; \
719-
} \
720-
if (dsn_unlikely(result.status() != dsn::http_status_code::kOk)) { \
721-
std::cout << "ERROR: send http request to query " << what << " metrics from node " \
722-
<< node.hp << " failed: " << dsn::get_http_status_message(result.status()) \
723-
<< std::endl \
724-
<< result.body() << std::endl; \
742+
const auto &res = process_get_metrics_result(result, node, what, ##__VA_ARGS__); \
743+
if (dsn_unlikely(!res)) { \
744+
fmt::println(res.description()); \
725745
return true; \
726746
} \
727747
} while (0)
728748

749+
// Adapt the result of some parsing operations on the metrics returned by `get_metrics` into the
750+
// structure that could be processed by `remote_command`.
751+
template <typename... Args>
752+
inline dsn::error_s process_parse_metrics_result(const dsn::error_s &result,
753+
const node_desc &node,
754+
const char *what,
755+
Args &&...args)
756+
{
757+
if (dsn_unlikely(!result)) {
758+
return FMT_ERR(result.code(),
759+
"ERROR: {} metrics response from node {} failed, msg={}",
760+
fmt::format(what, std::forward<Args>(args)...),
761+
node.hp,
762+
result);
763+
}
764+
765+
return dsn::error_s::ok();
766+
}
767+
729768
#define RETURN_SHELL_IF_PARSE_METRICS_FAILED(expr, node, what, ...) \
730769
do { \
731-
const auto &res = (expr); \
770+
const auto &res = process_parse_metrics_result(expr, node, what, ##__VA_ARGS__); \
732771
if (dsn_unlikely(!res)) { \
733-
std::cout << "ERROR: parse " << fmt::format(what, ##__VA_ARGS__) \
734-
<< " metrics response from node " << node.hp << " failed: " << res \
735-
<< std::endl; \
772+
fmt::println(res.description()); \
736773
return true; \
737774
} \
738775
} while (0)
@@ -832,37 +869,59 @@ class aggregate_stats_calcs
832869
}
833870

834871
// Create the aggregations as needed.
872+
DEF_CALC_CREATOR(assignments)
835873
DEF_CALC_CREATOR(sums)
836874
DEF_CALC_CREATOR(increases)
837875
DEF_CALC_CREATOR(rates)
838876

839877
#undef DEF_CALC_CREATOR
840878

879+
#define CALC_ASSIGNMENT_STATS(entities) \
880+
do { \
881+
if (_assignments) { \
882+
RETURN_NOT_OK(_assignments->assign(entities)); \
883+
} \
884+
} while (0)
885+
841886
#define CALC_ACCUM_STATS(entities) \
842887
do { \
843888
if (_sums) { \
844889
RETURN_NOT_OK(_sums->add_assign(entities)); \
845890
} \
846891
} while (0)
847892

848-
// Perform the chosen accum aggregations on the fetched metrics.
893+
// Perform the chosen aggregations (both assignment and accum) on the fetched metrics.
849894
dsn::error_s aggregate_metrics(const std::string &json_string)
850895
{
851896
DESERIALIZE_METRIC_QUERY_BRIEF_SNAPSHOT(value, json_string, query_snapshot);
852897

898+
return aggregate_metrics(query_snapshot);
899+
}
900+
901+
dsn::error_s aggregate_metrics(const dsn::metric_query_brief_value_snapshot &query_snapshot)
902+
{
903+
CALC_ASSIGNMENT_STATS(query_snapshot.entities);
853904
CALC_ACCUM_STATS(query_snapshot.entities);
854905

855906
return dsn::error_s::ok();
856907
}
857908

858-
// Perform all of the chosen aggregations (both accum and delta) on the fetched metrics.
909+
// Perform the chosen aggregations (assignement, accum, delta and rate) on the fetched metrics.
859910
dsn::error_s aggregate_metrics(const std::string &json_string_start,
860911
const std::string &json_string_end)
861912
{
862913
DESERIALIZE_METRIC_QUERY_BRIEF_2_SAMPLES(
863914
json_string_start, json_string_end, query_snapshot_start, query_snapshot_end);
864915

865-
// Apply ending sample to the accum aggregations.
916+
return aggregate_metrics(query_snapshot_start, query_snapshot_end);
917+
}
918+
919+
dsn::error_s
920+
aggregate_metrics(const dsn::metric_query_brief_value_snapshot &query_snapshot_start,
921+
const dsn::metric_query_brief_value_snapshot &query_snapshot_end)
922+
{
923+
// Apply ending sample to the assignment and accum aggregations.
924+
CALC_ASSIGNMENT_STATS(query_snapshot_end.entities);
866925
CALC_ACCUM_STATS(query_snapshot_end.entities);
867926

868927
const std::array deltas_list = {&_increases, &_rates};
@@ -884,9 +943,12 @@ class aggregate_stats_calcs
884943

885944
#undef CALC_ACCUM_STATS
886945

946+
#undef CALC_ASSIGNMENT_STATS
947+
887948
private:
888949
DISALLOW_COPY_AND_ASSIGN(aggregate_stats_calcs);
889950

951+
std::unique_ptr<aggregate_stats> _assignments;
890952
std::unique_ptr<aggregate_stats> _sums;
891953
std::unique_ptr<aggregate_stats> _increases;
892954
std::unique_ptr<aggregate_stats> _rates;
@@ -1940,7 +2002,7 @@ get_table_stats(shell_context *sc, uint32_t sample_interval_ms, std::vector<row_
19402002
RETURN_SHELL_IF_PARSE_METRICS_FAILED(
19412003
calcs->aggregate_metrics(results_start[i].body(), results_end[i].body()),
19422004
nodes[i],
1943-
"row data requests");
2005+
"aggregate row data requests");
19442006
}
19452007

19462008
return true;
@@ -1990,7 +2052,7 @@ inline bool get_partition_stats(shell_context *sc,
19902052
RETURN_SHELL_IF_PARSE_METRICS_FAILED(
19912053
calcs->aggregate_metrics(results_start[i].body(), results_end[i].body()),
19922054
nodes[i],
1993-
"row data requests for table(id={})",
2055+
"aggregate row data requests for table(id={})",
19942056
table_id);
19952057
}
19962058

src/shell/commands/data_operations.cpp

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@
6262
#include "utils/blob.h"
6363
#include "utils/defer.h"
6464
#include "utils/error_code.h"
65-
#include "utils/errors.h"
6665
#include "utils/flags.h"
6766
#include "utils/fmt_logging.h"
6867
#include "utils/metrics.h"
@@ -2294,7 +2293,7 @@ bool get_rdb_estimated_keys_stats(shell_context *sc,
22942293
create_rdb_estimated_keys_stats_calcs(table_id, pcs, nodes[i].hp, "replica", rows);
22952294
RETURN_SHELL_IF_PARSE_METRICS_FAILED(calcs->aggregate_metrics(results[i].body()),
22962295
nodes[i],
2297-
"rdb_estimated_keys for table(id={})",
2296+
"aggregate rdb_estimated_keys for table(id={})",
22982297
table_id);
22992298
}
23002299

0 commit comments

Comments
 (0)