metrics_logger_test.cc (15098B)
1 /* 2 * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 #include "api/test/metrics/metrics_logger.h" 11 12 #include <map> 13 #include <optional> 14 #include <string> 15 #include <vector> 16 17 #include "api/numerics/samples_stats_counter.h" 18 #include "api/test/metrics/metric.h" 19 #include "api/units/timestamp.h" 20 #include "system_wrappers/include/clock.h" 21 #include "test/gmock.h" 22 #include "test/gtest.h" 23 24 namespace webrtc { 25 namespace test { 26 namespace { 27 28 using ::testing::Eq; 29 using ::testing::IsEmpty; 30 using ::testing::SizeIs; 31 32 std::map<std::string, std::string> DefaultMetadata() { 33 return std::map<std::string, std::string>{{"key", "value"}}; 34 } 35 36 TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMetric) { 37 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 38 logger.LogSingleValueMetric( 39 "metric_name", "test_case_name", 40 /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 41 std::map<std::string, std::string>{{"key", "value"}}); 42 43 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 44 ASSERT_THAT(metrics, SizeIs(1)); 45 const Metric& metric = metrics[0]; 46 EXPECT_THAT(metric.name, Eq("metric_name")); 47 EXPECT_THAT(metric.test_case, Eq("test_case_name")); 48 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); 49 EXPECT_THAT(metric.improvement_direction, 50 Eq(ImprovementDirection::kBiggerIsBetter)); 51 EXPECT_THAT(metric.metric_metadata, 52 Eq(std::map<std::string, std::string>{{"key", "value"}})); 53 ASSERT_THAT(metric.time_series.samples, SizeIs(1)); 54 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); 55 EXPECT_THAT(metric.time_series.samples[0].sample_metadata, 56 Eq(std::map<std::string, std::string>{})); 57 ASSERT_THAT(metric.stats.mean, std::optional<double>(10.0)); 58 ASSERT_THAT(metric.stats.stddev, std::nullopt); 59 ASSERT_THAT(metric.stats.min, std::optional<double>(10.0)); 60 ASSERT_THAT(metric.stats.max, std::optional<double>(10.0)); 61 } 62 63 TEST(DefaultMetricsLoggerTest, LogMetricWithSamplesStatsCounterRecordsMetric) { 64 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 65 66 SamplesStatsCounter values; 67 values.AddSample(SamplesStatsCounter::StatsSample{ 68 .value = 10, 69 .time = Clock::GetRealTimeClock()->CurrentTime(), 70 .metadata = 71 std::map<std::string, std::string>{{"point_key1", "value1"}}}); 72 values.AddSample(SamplesStatsCounter::StatsSample{ 73 .value = 20, 74 .time = Clock::GetRealTimeClock()->CurrentTime(), 75 .metadata = 76 std::map<std::string, std::string>{{"point_key2", "value2"}}}); 77 logger.LogMetric("metric_name", "test_case_name", values, Unit::kMilliseconds, 78 ImprovementDirection::kBiggerIsBetter, 79 std::map<std::string, std::string>{{"key", "value"}}); 80 81 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 82 ASSERT_THAT(metrics, SizeIs(1)); 83 const Metric& metric = metrics[0]; 84 EXPECT_THAT(metric.name, Eq("metric_name")); 85 EXPECT_THAT(metric.test_case, Eq("test_case_name")); 86 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); 87 EXPECT_THAT(metric.improvement_direction, 88 Eq(ImprovementDirection::kBiggerIsBetter)); 89 EXPECT_THAT(metric.metric_metadata, 90 Eq(std::map<std::string, std::string>{{"key", "value"}})); 91 ASSERT_THAT(metric.time_series.samples, SizeIs(2)); 92 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); 93 EXPECT_THAT(metric.time_series.samples[0].sample_metadata, 94 Eq(std::map<std::string, std::string>{{"point_key1", "value1"}})); 95 EXPECT_THAT(metric.time_series.samples[1].value, Eq(20.0)); 96 EXPECT_THAT(metric.time_series.samples[1].sample_metadata, 97 Eq(std::map<std::string, std::string>{{"point_key2", "value2"}})); 98 ASSERT_THAT(metric.stats.mean, std::optional<double>(15.0)); 99 ASSERT_THAT(metric.stats.stddev, std::optional<double>(5.0)); 100 ASSERT_THAT(metric.stats.min, std::optional<double>(10.0)); 101 ASSERT_THAT(metric.stats.max, std::optional<double>(20.0)); 102 } 103 104 TEST(DefaultMetricsLoggerTest, 105 LogMetricWithEmptySamplesStatsCounterRecordsEmptyMetric) { 106 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 107 SamplesStatsCounter values; 108 logger.LogMetric("metric_name", "test_case_name", values, Unit::kUnitless, 109 ImprovementDirection::kBiggerIsBetter, DefaultMetadata()); 110 111 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 112 ASSERT_THAT(metrics, SizeIs(1)); 113 EXPECT_THAT(metrics[0].name, Eq("metric_name")); 114 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name")); 115 EXPECT_THAT(metrics[0].time_series.samples, IsEmpty()); 116 ASSERT_THAT(metrics[0].stats.mean, Eq(std::nullopt)); 117 ASSERT_THAT(metrics[0].stats.stddev, Eq(std::nullopt)); 118 ASSERT_THAT(metrics[0].stats.min, Eq(std::nullopt)); 119 ASSERT_THAT(metrics[0].stats.max, Eq(std::nullopt)); 120 } 121 122 TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMetric) { 123 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 124 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; 125 logger.LogMetric("metric_name", "test_case_name", metric_stats, 126 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 127 std::map<std::string, std::string>{{"key", "value"}}); 128 129 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 130 ASSERT_THAT(metrics, SizeIs(1)); 131 const Metric& metric = metrics[0]; 132 EXPECT_THAT(metric.name, Eq("metric_name")); 133 EXPECT_THAT(metric.test_case, Eq("test_case_name")); 134 EXPECT_THAT(metric.unit, Eq(Unit::kMilliseconds)); 135 EXPECT_THAT(metric.improvement_direction, 136 Eq(ImprovementDirection::kBiggerIsBetter)); 137 EXPECT_THAT(metric.metric_metadata, 138 Eq(std::map<std::string, std::string>{{"key", "value"}})); 139 ASSERT_THAT(metric.time_series.samples, IsEmpty()); 140 ASSERT_THAT(metric.stats.mean, std::optional<double>(15.0)); 141 ASSERT_THAT(metric.stats.stddev, std::optional<double>(5.0)); 142 ASSERT_THAT(metric.stats.min, std::optional<double>(10.0)); 143 ASSERT_THAT(metric.stats.max, std::optional<double>(20.0)); 144 } 145 146 TEST(DefaultMetricsLoggerTest, LogSingleValueMetricRecordsMultipleMetrics) { 147 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 148 149 logger.LogSingleValueMetric("metric_name1", "test_case_name1", 150 /*value=*/10, Unit::kMilliseconds, 151 ImprovementDirection::kBiggerIsBetter, 152 DefaultMetadata()); 153 logger.LogSingleValueMetric("metric_name2", "test_case_name2", 154 /*value=*/10, Unit::kMilliseconds, 155 ImprovementDirection::kBiggerIsBetter, 156 DefaultMetadata()); 157 158 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 159 ASSERT_THAT(metrics, SizeIs(2)); 160 EXPECT_THAT(metrics[0].name, Eq("metric_name1")); 161 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); 162 EXPECT_THAT(metrics[1].name, Eq("metric_name2")); 163 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); 164 } 165 166 TEST(DefaultMetricsLoggerTest, 167 LogMetricWithSamplesStatsCounterRecordsMultipleMetrics) { 168 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 169 SamplesStatsCounter values; 170 values.AddSample(SamplesStatsCounter::StatsSample{ 171 .value = 10, 172 .time = Clock::GetRealTimeClock()->CurrentTime(), 173 .metadata = DefaultMetadata()}); 174 values.AddSample(SamplesStatsCounter::StatsSample{ 175 .value = 20, 176 .time = Clock::GetRealTimeClock()->CurrentTime(), 177 .metadata = DefaultMetadata()}); 178 179 logger.LogMetric("metric_name1", "test_case_name1", values, 180 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 181 DefaultMetadata()); 182 logger.LogMetric("metric_name2", "test_case_name2", values, 183 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 184 DefaultMetadata()); 185 186 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 187 ASSERT_THAT(metrics, SizeIs(2)); 188 EXPECT_THAT(metrics[0].name, Eq("metric_name1")); 189 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); 190 EXPECT_THAT(metrics[1].name, Eq("metric_name2")); 191 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); 192 } 193 194 TEST(DefaultMetricsLoggerTest, LogMetricWithStatsRecordsMultipleMetrics) { 195 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 196 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; 197 198 logger.LogMetric("metric_name1", "test_case_name1", metric_stats, 199 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 200 DefaultMetadata()); 201 logger.LogMetric("metric_name2", "test_case_name2", metric_stats, 202 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 203 DefaultMetadata()); 204 205 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 206 ASSERT_THAT(metrics, SizeIs(2)); 207 EXPECT_THAT(metrics[0].name, Eq("metric_name1")); 208 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); 209 EXPECT_THAT(metrics[1].name, Eq("metric_name2")); 210 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); 211 } 212 213 TEST(DefaultMetricsLoggerTest, 214 LogMetricThroughtAllMethodsAccumulateAllMetrics) { 215 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 216 SamplesStatsCounter values; 217 values.AddSample(SamplesStatsCounter::StatsSample{ 218 .value = 10, 219 .time = Clock::GetRealTimeClock()->CurrentTime(), 220 .metadata = DefaultMetadata()}); 221 values.AddSample(SamplesStatsCounter::StatsSample{ 222 .value = 20, 223 .time = Clock::GetRealTimeClock()->CurrentTime(), 224 .metadata = DefaultMetadata()}); 225 Metric::Stats metric_stats{.mean = 15, .stddev = 5, .min = 10, .max = 20}; 226 227 logger.LogSingleValueMetric("metric_name1", "test_case_name1", 228 /*value=*/10, Unit::kMilliseconds, 229 ImprovementDirection::kBiggerIsBetter, 230 DefaultMetadata()); 231 logger.LogMetric("metric_name2", "test_case_name2", values, 232 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 233 DefaultMetadata()); 234 logger.LogMetric("metric_name3", "test_case_name3", metric_stats, 235 Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 236 DefaultMetadata()); 237 238 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 239 ASSERT_THAT(metrics.size(), Eq(3lu)); 240 EXPECT_THAT(metrics[0].name, Eq("metric_name1")); 241 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name1")); 242 EXPECT_THAT(metrics[1].name, Eq("metric_name2")); 243 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name2")); 244 EXPECT_THAT(metrics[2].name, Eq("metric_name3")); 245 EXPECT_THAT(metrics[2].test_case, Eq("test_case_name3")); 246 } 247 248 TEST(DefaultMetricsLoggerTest, AccumulatedMetricsReturnedInCollectedMetrics) { 249 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 250 logger.GetMetricsAccumulator()->AddSample( 251 "metric_name", "test_case_name", 252 /*value=*/10, Timestamp::Seconds(1), 253 /*point_metadata=*/std::map<std::string, std::string>{{"key", "value"}}); 254 255 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 256 ASSERT_THAT(metrics, SizeIs(1)); 257 const Metric& metric = metrics[0]; 258 EXPECT_THAT(metric.name, Eq("metric_name")); 259 EXPECT_THAT(metric.test_case, Eq("test_case_name")); 260 EXPECT_THAT(metric.unit, Eq(Unit::kUnitless)); 261 EXPECT_THAT(metric.improvement_direction, 262 Eq(ImprovementDirection::kNeitherIsBetter)); 263 EXPECT_THAT(metric.metric_metadata, IsEmpty()); 264 ASSERT_THAT(metric.time_series.samples, SizeIs(1)); 265 EXPECT_THAT(metric.time_series.samples[0].value, Eq(10.0)); 266 EXPECT_THAT(metric.time_series.samples[0].timestamp, 267 Eq(Timestamp::Seconds(1))); 268 EXPECT_THAT(metric.time_series.samples[0].sample_metadata, 269 Eq(std::map<std::string, std::string>{{"key", "value"}})); 270 ASSERT_THAT(metric.stats.mean, std::optional<double>(10.0)); 271 ASSERT_THAT(metric.stats.stddev, std::optional<double>(0.0)); 272 ASSERT_THAT(metric.stats.min, std::optional<double>(10.0)); 273 ASSERT_THAT(metric.stats.max, std::optional<double>(10.0)); 274 } 275 276 TEST(DefaultMetricsLoggerTest, 277 AccumulatedMetricsReturnedTogetherWithLoggedMetrics) { 278 DefaultMetricsLogger logger(Clock::GetRealTimeClock()); 279 logger.LogSingleValueMetric( 280 "metric_name1", "test_case_name1", 281 /*value=*/10, Unit::kMilliseconds, ImprovementDirection::kBiggerIsBetter, 282 std::map<std::string, std::string>{{"key_m", "value_m"}}); 283 logger.GetMetricsAccumulator()->AddSample( 284 "metric_name2", "test_case_name2", 285 /*value=*/10, Timestamp::Seconds(1), 286 /*point_metadata=*/ 287 std::map<std::string, std::string>{{"key_s", "value_s"}}); 288 289 std::vector<Metric> metrics = logger.GetCollectedMetrics(); 290 ASSERT_THAT(metrics, SizeIs(2)); 291 EXPECT_THAT(metrics[0].name, Eq("metric_name2")); 292 EXPECT_THAT(metrics[0].test_case, Eq("test_case_name2")); 293 EXPECT_THAT(metrics[0].unit, Eq(Unit::kUnitless)); 294 EXPECT_THAT(metrics[0].improvement_direction, 295 Eq(ImprovementDirection::kNeitherIsBetter)); 296 EXPECT_THAT(metrics[0].metric_metadata, IsEmpty()); 297 ASSERT_THAT(metrics[0].time_series.samples, SizeIs(1)); 298 EXPECT_THAT(metrics[0].time_series.samples[0].value, Eq(10.0)); 299 EXPECT_THAT(metrics[0].time_series.samples[0].timestamp, 300 Eq(Timestamp::Seconds(1))); 301 EXPECT_THAT(metrics[0].time_series.samples[0].sample_metadata, 302 Eq(std::map<std::string, std::string>{{"key_s", "value_s"}})); 303 ASSERT_THAT(metrics[0].stats.mean, std::optional<double>(10.0)); 304 ASSERT_THAT(metrics[0].stats.stddev, std::optional<double>(0.0)); 305 ASSERT_THAT(metrics[0].stats.min, std::optional<double>(10.0)); 306 ASSERT_THAT(metrics[0].stats.max, std::optional<double>(10.0)); 307 EXPECT_THAT(metrics[1].name, Eq("metric_name1")); 308 EXPECT_THAT(metrics[1].test_case, Eq("test_case_name1")); 309 EXPECT_THAT(metrics[1].unit, Eq(Unit::kMilliseconds)); 310 EXPECT_THAT(metrics[1].improvement_direction, 311 Eq(ImprovementDirection::kBiggerIsBetter)); 312 EXPECT_THAT(metrics[1].metric_metadata, 313 Eq(std::map<std::string, std::string>{{"key_m", "value_m"}})); 314 ASSERT_THAT(metrics[1].time_series.samples, SizeIs(1)); 315 EXPECT_THAT(metrics[1].time_series.samples[0].value, Eq(10.0)); 316 EXPECT_THAT(metrics[1].time_series.samples[0].sample_metadata, 317 Eq(std::map<std::string, std::string>{})); 318 ASSERT_THAT(metrics[1].stats.mean, std::optional<double>(10.0)); 319 ASSERT_THAT(metrics[1].stats.stddev, std::nullopt); 320 ASSERT_THAT(metrics[1].stats.min, std::optional<double>(10.0)); 321 ASSERT_THAT(metrics[1].stats.max, std::optional<double>(10.0)); 322 } 323 324 } // namespace 325 } // namespace test 326 } // namespace webrtc