test_oos.c (12006B)
1 /* Copyright (c) 2016-2021, The Tor Project, Inc. */ 2 /* See LICENSE for licensing information */ 3 4 /* Unit tests for OOS handler */ 5 6 #define CONNECTION_PRIVATE 7 8 #include "core/or/or.h" 9 #include "app/config/config.h" 10 #include "core/mainloop/connection.h" 11 #include "core/or/connection_or.h" 12 #include "feature/dircommon/directory.h" 13 #include "core/mainloop/mainloop.h" 14 #include "test/test.h" 15 16 #include "feature/dircommon/dir_connection_st.h" 17 #include "core/or/or_connection_st.h" 18 19 static or_options_t mock_options; 20 21 static void 22 reset_options_mock(void) 23 { 24 memset(&mock_options, 0, sizeof(or_options_t)); 25 } 26 27 static const or_options_t * 28 mock_get_options(void) 29 { 30 return &mock_options; 31 } 32 33 static int moribund_calls = 0; 34 static int moribund_conns = 0; 35 36 static int 37 mock_connection_count_moribund(void) 38 { 39 ++moribund_calls; 40 41 return moribund_conns; 42 } 43 44 /* 45 * For unit test purposes it's sufficient to tell that 46 * kill_conn_list_for_oos() was called with an approximately 47 * sane argument; it's just the thing we returned from the 48 * mock for pick_oos_victims(). 49 */ 50 51 static int kill_conn_list_calls = 0; 52 static int kill_conn_list_killed = 0; 53 54 static void 55 kill_conn_list_mock(smartlist_t *conns) 56 { 57 ++kill_conn_list_calls; 58 59 tt_ptr_op(conns, OP_NE, NULL); 60 61 kill_conn_list_killed += smartlist_len(conns); 62 63 done: 64 return; 65 } 66 67 static int pick_oos_mock_calls = 0; 68 static int pick_oos_mock_fail = 0; 69 static int pick_oos_mock_last_n = 0; 70 71 static smartlist_t * 72 pick_oos_victims_mock(int n) 73 { 74 smartlist_t *l = NULL; 75 int i; 76 77 ++pick_oos_mock_calls; 78 79 tt_int_op(n, OP_GT, 0); 80 81 if (!pick_oos_mock_fail) { 82 /* 83 * connection_check_oos() just passes the list onto 84 * kill_conn_list_for_oos(); we don't need to simulate 85 * its content for this mock, just its existence, but 86 * we do need to check the parameter. 87 */ 88 l = smartlist_new(); 89 for (i = 0; i < n; ++i) smartlist_add(l, NULL); 90 } else { 91 l = NULL; 92 } 93 94 pick_oos_mock_last_n = n; 95 96 done: 97 return l; 98 } 99 100 /** Unit test for the logic in connection_check_oos(), which is concerned 101 * with comparing thresholds and connection counts to decide if an OOS has 102 * occurred and if so, how many connections to try to kill, and then using 103 * pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim 104 * duty. 105 */ 106 static void 107 test_oos_connection_check_oos(void *arg) 108 { 109 (void)arg; 110 111 /* Set up mocks */ 112 reset_options_mock(); 113 /* OOS handling is only sensitive to these fields */ 114 mock_options.ConnLimit = 32; 115 mock_options.ConnLimit_ = 64; 116 mock_options.ConnLimit_high_thresh = 60; 117 mock_options.ConnLimit_low_thresh = 50; 118 MOCK(get_options, mock_get_options); 119 moribund_calls = 0; 120 moribund_conns = 0; 121 MOCK(connection_count_moribund, mock_connection_count_moribund); 122 kill_conn_list_calls = 0; 123 kill_conn_list_killed = 0; 124 MOCK(kill_conn_list_for_oos, kill_conn_list_mock); 125 pick_oos_mock_calls = 0; 126 pick_oos_mock_fail = 0; 127 MOCK(pick_oos_victims, pick_oos_victims_mock); 128 129 /* No OOS case */ 130 connection_check_oos(50, 0); 131 tt_int_op(moribund_calls, OP_EQ, 0); 132 tt_int_op(pick_oos_mock_calls, OP_EQ, 0); 133 tt_int_op(kill_conn_list_calls, OP_EQ, 0); 134 135 /* OOS from socket count, nothing moribund */ 136 connection_check_oos(62, 0); 137 tt_int_op(moribund_calls, OP_EQ, 1); 138 tt_int_op(pick_oos_mock_calls, OP_EQ, 1); 139 /* 12 == 62 - ConnLimit_low_thresh */ 140 tt_int_op(pick_oos_mock_last_n, OP_EQ, 12); 141 tt_int_op(kill_conn_list_calls, OP_EQ, 1); 142 tt_int_op(kill_conn_list_killed, OP_EQ, 12); 143 144 /* OOS from socket count, some are moribund */ 145 kill_conn_list_killed = 0; 146 moribund_conns = 5; 147 connection_check_oos(62, 0); 148 tt_int_op(moribund_calls, OP_EQ, 2); 149 tt_int_op(pick_oos_mock_calls, OP_EQ, 2); 150 /* 7 == 62 - ConnLimit_low_thresh - moribund_conns */ 151 tt_int_op(pick_oos_mock_last_n, OP_EQ, 7); 152 tt_int_op(kill_conn_list_calls, OP_EQ, 2); 153 tt_int_op(kill_conn_list_killed, OP_EQ, 7); 154 155 /* OOS from socket count, but pick fails */ 156 kill_conn_list_killed = 0; 157 moribund_conns = 0; 158 pick_oos_mock_fail = 1; 159 connection_check_oos(62, 0); 160 tt_int_op(moribund_calls, OP_EQ, 3); 161 tt_int_op(pick_oos_mock_calls, OP_EQ, 3); 162 tt_int_op(kill_conn_list_calls, OP_EQ, 2); 163 tt_int_op(kill_conn_list_killed, OP_EQ, 0); 164 pick_oos_mock_fail = 0; 165 166 /* 167 * OOS from socket count with so many moribund conns 168 * we have none to kill. 169 */ 170 kill_conn_list_killed = 0; 171 moribund_conns = 15; 172 connection_check_oos(62, 0); 173 tt_int_op(moribund_calls, OP_EQ, 4); 174 tt_int_op(pick_oos_mock_calls, OP_EQ, 3); 175 tt_int_op(kill_conn_list_calls, OP_EQ, 2); 176 177 /* 178 * OOS from socket exhaustion; OOS handler will try to 179 * kill 1/10 (5) of the connections. 180 */ 181 kill_conn_list_killed = 0; 182 moribund_conns = 0; 183 connection_check_oos(50, 1); 184 tt_int_op(moribund_calls, OP_EQ, 5); 185 tt_int_op(pick_oos_mock_calls, OP_EQ, 4); 186 tt_int_op(kill_conn_list_calls, OP_EQ, 3); 187 tt_int_op(kill_conn_list_killed, OP_EQ, 5); 188 189 /* OOS from socket exhaustion with moribund conns */ 190 kill_conn_list_killed = 0; 191 moribund_conns = 2; 192 connection_check_oos(50, 1); 193 tt_int_op(moribund_calls, OP_EQ, 6); 194 tt_int_op(pick_oos_mock_calls, OP_EQ, 5); 195 tt_int_op(kill_conn_list_calls, OP_EQ, 4); 196 tt_int_op(kill_conn_list_killed, OP_EQ, 3); 197 198 /* OOS from socket exhaustion with many moribund conns */ 199 kill_conn_list_killed = 0; 200 moribund_conns = 7; 201 connection_check_oos(50, 1); 202 tt_int_op(moribund_calls, OP_EQ, 7); 203 tt_int_op(pick_oos_mock_calls, OP_EQ, 5); 204 tt_int_op(kill_conn_list_calls, OP_EQ, 4); 205 206 /* OOS with both socket exhaustion and above-threshold */ 207 kill_conn_list_killed = 0; 208 moribund_conns = 0; 209 connection_check_oos(62, 1); 210 tt_int_op(moribund_calls, OP_EQ, 8); 211 tt_int_op(pick_oos_mock_calls, OP_EQ, 6); 212 tt_int_op(kill_conn_list_calls, OP_EQ, 5); 213 tt_int_op(kill_conn_list_killed, OP_EQ, 12); 214 215 /* 216 * OOS with both socket exhaustion and above-threshold with some 217 * moribund conns 218 */ 219 kill_conn_list_killed = 0; 220 moribund_conns = 5; 221 connection_check_oos(62, 1); 222 tt_int_op(moribund_calls, OP_EQ, 9); 223 tt_int_op(pick_oos_mock_calls, OP_EQ, 7); 224 tt_int_op(kill_conn_list_calls, OP_EQ, 6); 225 tt_int_op(kill_conn_list_killed, OP_EQ, 7); 226 227 /* 228 * OOS with both socket exhaustion and above-threshold with many 229 * moribund conns 230 */ 231 kill_conn_list_killed = 0; 232 moribund_conns = 15; 233 connection_check_oos(62, 1); 234 tt_int_op(moribund_calls, OP_EQ, 10); 235 tt_int_op(pick_oos_mock_calls, OP_EQ, 7); 236 tt_int_op(kill_conn_list_calls, OP_EQ, 6); 237 238 done: 239 240 UNMOCK(pick_oos_victims); 241 UNMOCK(kill_conn_list_for_oos); 242 UNMOCK(connection_count_moribund); 243 UNMOCK(get_options); 244 245 return; 246 } 247 248 static int cfe_calls = 0; 249 250 static void 251 close_for_error_mock(or_connection_t *orconn, int flush) 252 { 253 (void)flush; 254 255 tt_ptr_op(orconn, OP_NE, NULL); 256 ++cfe_calls; 257 258 done: 259 return; 260 } 261 262 static int mark_calls = 0; 263 264 static void 265 mark_for_close_oos_mock(connection_t *conn, 266 int line, const char *file) 267 { 268 (void)line; 269 (void)file; 270 271 tt_ptr_op(conn, OP_NE, NULL); 272 ++mark_calls; 273 274 done: 275 return; 276 } 277 278 static void 279 test_oos_kill_conn_list(void *arg) 280 { 281 connection_t *c1, *c2; 282 or_connection_t *or_c1 = NULL; 283 dir_connection_t *dir_c2 = NULL; 284 smartlist_t *l = NULL; 285 (void)arg; 286 287 /* Set up mocks */ 288 mark_calls = 0; 289 MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock); 290 cfe_calls = 0; 291 MOCK(connection_or_close_for_error, close_for_error_mock); 292 293 /* Make fake conns */ 294 or_c1 = tor_malloc_zero(sizeof(*or_c1)); 295 or_c1->base_.magic = OR_CONNECTION_MAGIC; 296 or_c1->base_.type = CONN_TYPE_OR; 297 c1 = TO_CONN(or_c1); 298 dir_c2 = tor_malloc_zero(sizeof(*dir_c2)); 299 dir_c2->base_.magic = DIR_CONNECTION_MAGIC; 300 dir_c2->base_.type = CONN_TYPE_DIR; 301 dir_c2->base_.state = DIR_CONN_STATE_MIN_; 302 dir_c2->base_.purpose = DIR_PURPOSE_MIN_; 303 c2 = TO_CONN(dir_c2); 304 305 tt_ptr_op(c1, OP_NE, NULL); 306 tt_ptr_op(c2, OP_NE, NULL); 307 308 /* Make list */ 309 l = smartlist_new(); 310 smartlist_add(l, c1); 311 smartlist_add(l, c2); 312 313 /* Run kill_conn_list_for_oos() */ 314 kill_conn_list_for_oos(l); 315 316 /* Check call counters */ 317 tt_int_op(mark_calls, OP_EQ, 1); 318 tt_int_op(cfe_calls, OP_EQ, 1); 319 320 done: 321 322 UNMOCK(connection_or_close_for_error); 323 UNMOCK(connection_mark_for_close_internal_); 324 325 if (l) smartlist_free(l); 326 tor_free(or_c1); 327 tor_free(dir_c2); 328 329 return; 330 } 331 332 static smartlist_t *conns_for_mock = NULL; 333 334 static smartlist_t * 335 get_conns_mock(void) 336 { 337 return conns_for_mock; 338 } 339 340 /* 341 * For this mock, we pretend all conns have either zero or one circuits, 342 * depending on if this appears on the list of things to say have a circuit. 343 */ 344 345 static smartlist_t *conns_with_circs = NULL; 346 347 static int 348 get_num_circuits_mock(or_connection_t *conn) 349 { 350 int circs = 0; 351 352 tt_ptr_op(conn, OP_NE, NULL); 353 354 if (conns_with_circs && 355 smartlist_contains(conns_with_circs, TO_CONN(conn))) { 356 circs = 1; 357 } 358 359 done: 360 return circs; 361 } 362 363 static void 364 test_oos_pick_oos_victims(void *arg) 365 { 366 (void)arg; 367 or_connection_t *ortmp; 368 dir_connection_t *dirtmp; 369 smartlist_t *picked; 370 371 /* Set up mocks */ 372 conns_for_mock = smartlist_new(); 373 MOCK(get_connection_array, get_conns_mock); 374 conns_with_circs = smartlist_new(); 375 MOCK(connection_or_get_num_circuits, get_num_circuits_mock); 376 377 /* Make some fake connections */ 378 ortmp = tor_malloc_zero(sizeof(*ortmp)); 379 ortmp->base_.magic = OR_CONNECTION_MAGIC; 380 ortmp->base_.type = CONN_TYPE_OR; 381 smartlist_add(conns_for_mock, TO_CONN(ortmp)); 382 /* We'll pretend this one has a circuit too */ 383 smartlist_add(conns_with_circs, TO_CONN(ortmp)); 384 /* Next one */ 385 ortmp = tor_malloc_zero(sizeof(*ortmp)); 386 ortmp->base_.magic = OR_CONNECTION_MAGIC; 387 ortmp->base_.type = CONN_TYPE_OR; 388 smartlist_add(conns_for_mock, TO_CONN(ortmp)); 389 /* Next one is moribund */ 390 ortmp = tor_malloc_zero(sizeof(*ortmp)); 391 ortmp->base_.magic = OR_CONNECTION_MAGIC; 392 ortmp->base_.type = CONN_TYPE_OR; 393 ortmp->base_.marked_for_close = 1; 394 smartlist_add(conns_for_mock, TO_CONN(ortmp)); 395 /* Last one isn't an orconn */ 396 dirtmp = tor_malloc_zero(sizeof(*dirtmp)); 397 dirtmp->base_.magic = DIR_CONNECTION_MAGIC; 398 dirtmp->base_.type = CONN_TYPE_DIR; 399 smartlist_add(conns_for_mock, TO_CONN(dirtmp)); 400 401 /* Try picking one */ 402 picked = pick_oos_victims(1); 403 /* It should be the one with circuits */ 404 tt_ptr_op(picked, OP_NE, NULL); 405 tt_int_op(smartlist_len(picked), OP_EQ, 1); 406 tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0))); 407 smartlist_free(picked); 408 409 /* Try picking none */ 410 picked = pick_oos_victims(0); 411 /* We should get an empty list */ 412 tt_ptr_op(picked, OP_NE, NULL); 413 tt_int_op(smartlist_len(picked), OP_EQ, 0); 414 smartlist_free(picked); 415 416 /* Try picking two */ 417 picked = pick_oos_victims(2); 418 /* We should get both active orconns */ 419 tt_ptr_op(picked, OP_NE, NULL); 420 tt_int_op(smartlist_len(picked), OP_EQ, 2); 421 tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0))); 422 tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1))); 423 smartlist_free(picked); 424 425 /* Try picking three - only two are eligible */ 426 picked = pick_oos_victims(3); 427 tt_int_op(smartlist_len(picked), OP_EQ, 2); 428 tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0))); 429 tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1))); 430 smartlist_free(picked); 431 432 done: 433 434 /* Free leftover stuff */ 435 if (conns_with_circs) { 436 smartlist_free(conns_with_circs); 437 conns_with_circs = NULL; 438 } 439 440 UNMOCK(connection_or_get_num_circuits); 441 442 if (conns_for_mock) { 443 SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c)); 444 smartlist_free(conns_for_mock); 445 conns_for_mock = NULL; 446 } 447 448 UNMOCK(get_connection_array); 449 450 return; 451 } 452 453 struct testcase_t oos_tests[] = { 454 { "connection_check_oos", test_oos_connection_check_oos, 455 TT_FORK, NULL, NULL }, 456 { "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL }, 457 { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL }, 458 END_OF_TESTCASES 459 };