src/main/connection.c

   1 /*
   2  *   This program is free software; you can redistribute it and/or modify
   3  *   it under the terms of the GNU General Public License as published by
   4  *   the Free Software Foundation; either version 2 of the License, or
   5  *   (at your option) any later version.
   6  *
   7  *   This program is distributed in the hope that it will be useful,
   8  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  *   GNU General Public License for more details.
  11  *
  12  *   You should have received a copy of the GNU General Public License
  13  *   along with this program; if not, write to the Free Software
  14  *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  15  */
  16
  17 /**
  18  * @file connection.c
  19  * @brief Handle pools of connections (threads, sockets, etc.)
  20  * @note This API must be used by all modules in the public distribution that
  21  * maintain pools of connections.
  22  *
  23  * @copyright 2012  The FreeRADIUS server project
  24  * @copyright 2012  Alan DeKok <aland@deployingradius.com>
  25  */
  26 RCSID("$Id$")
  27
  28 #include <freeradius-devel/radiusd.h>
  29 #include <freeradius-devel/rad_assert.h>
  30
  31 typedef struct fr_connection fr_connection_t;
  32
  33 static int fr_connection_pool_check(fr_connection_pool_t *pool);
  34
  35 /** An individual connection within the connection pool
  36  *
  37  * Defines connection counters, timestamps, and holds a pointer to the
  38  * connection handle itself.
  39  *
  40  * @see fr_connection_pool_t
  41  */
  42 struct fr_connection {
  43         fr_connection_t *prev;          //!< Previous connection in list.
  44         fr_connection_t *next;          //!< Next connection in list.
  45
  46         time_t          created;        //!< Time connection was created.
  47         time_t          last_used;      //!< Last time the connection was
  48                                         //!< reserved.
  49
  50         uint64_t        num_uses;       //!< Number of times the connection
  51                                         //!< has been reserved.
  52         int             in_use;         //!< Whether the connection is currently
  53                                         //!< reserved.
  54         uint64_t        number;         //!< Unique ID assigned when the
  55                                         //!< connection is created, these will
  56                                         //!< monotonically increase over the
  57                                         //!< lifetime of the connection pool.
  58         void            *connection;    //!< Pointer to whatever the module
  59                                         //!< uses for a connection handle.
  60 };
  61
  62 /** A connection pool
  63  *
  64  * Defines the configuration of the connection pool, all the counters and
  65  * timestamps related to the connection pool, the mutex that stops multiple
  66  * threads leaving the pool in an inconsistent state, and the callbacks
  67  * required to open, close and check the status of connections within the pool.
  68  *
  69  * @see fr_connection
  70  */
  71 struct fr_connection_pool_t {
  72         int             start;          //!< Number of initial connections
  73         int             min;            //!< Minimum number of concurrent
  74                                         //!< connections to keep open.
  75         int             max;            //!< Maximum number of concurrent
  76                                         //!< connections to allow.
  77         int             spare;          //!< Number of spare connections to try
  78                                         //!< and maintain.
  79         int             cleanup_delay;  //!< How long a connection can go unused
  80                                         //!< for before it's closed
  81                                         //!< (0 is infinite).
  82         uint64_t        max_uses;       //!< Maximum number of times a
  83                                         //!< connection can be used before being
  84                                         //!< closed.
  85         int             lifetime;       //!< How long a connection can be open
  86                                         //!< before being closed (irrespective
  87                                         //!< of whether it's idle or not).
  88         int             idle_timeout;   //!< How long a connection can be idle
  89                                         //!< before being closed.
  90
  91         bool            trigger;        //!< If true execute connection triggers
  92                                         //!< associated with the connection
  93                                         //!< pool.
  94
  95         bool            spread;         //!< If true requests will be spread
  96                                         //!< across all connections, instead of
  97                                         //!< re-using the most recently used
  98                                         //!< connections first.
  99
 100         time_t          last_checked;   //!< Last time we pruned the connection
 101                                         //!< pool.
 102         time_t          last_spawned;   //!< Last time we spawned a connection.
 103         time_t          last_failed;    //!< Last time we tried to spawn a
 104                                         //!< a connection but failed.
 105         time_t          last_complained;//!< Last time we complained about
 106                                         //!< configuration parameters.
 107         time_t          last_throttled; //!< Last time we refused to spawn a
 108                                         //!< connection because the last
 109                                         //!< connection failed, or we were
 110                                         //!< already spawning a connection.
 111         time_t          last_at_max;    //!< Last time we hit the maximum number
 112                                         //!< of allowed connections.
 113
 114         uint64_t        count;          //!< Number of connections spawned over
 115                                         //!< the lifetime of the pool.
 116         int             num;            //!< Number of connections in the pool.
 117         int             active;         //!< Number of currently reserved
 118                                         //!< connections.
 119
 120         fr_connection_t *head;          //!< Start of the connection list.
 121         fr_connection_t *tail;          //!< End of the connection list.
 122
 123         bool            spawning;       //!< Whether we are currently attempting
 124                                         //!< to spawn a new connection.
 125
 126 #ifdef HAVE_PTHREAD_H
 127         pthread_mutex_t mutex;          //!< Mutex used to keep consistent state
 128                                         //!< when making modifications in
 129                                         //!< threaded mode.
 130 #endif
 131
 132         CONF_SECTION    *cs;            //!< Configuration section holding
 133                                         //!< the section of parsed config file
 134                                         //!< that relates to this pool.
 135         void            *ctx;           //!< Pointer to context data that will
 136                                         //!< be passed to callbacks.
 137
 138         char            *log_prefix;    //!< Log prefix to prepend to all log
 139                                         //!< messages created by the connection
 140                                         //!< pool code.
 141
 142         fr_connection_create_t  create; //!< Function used to create new
 143                                         //!< connections.
 144         fr_connection_alive_t   alive;  //!< Function used to check status
 145                                         //!< of connections.
 146         fr_connection_delete_t  delete; //!< Function used to close existing
 147                                         //!< connections.
 148 };
 149
 150 #define LOG_PREFIX "rlm_%s (%s)"
 151 #ifndef HAVE_PTHREAD_H
 152 #define pthread_mutex_lock(_x)
 153 #define pthread_mutex_unlock(_x)
 154 #endif
 155
 156 static const CONF_PARSER connection_config[] = {
 157         { "start",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, start),
 158           0, "5" },
 159         { "min",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, min),
 160           0, "5" },
 161         { "max",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max),
 162           0, "10" },
 163         { "spare",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, spare),
 164           0, "3" },
 165         { "uses",     PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max_uses),
 166           0, "0" },
 167         { "lifetime", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, lifetime),
 168           0, "0" },
 169         { "cleanup_delay", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, cleanup_delay),
 170           0, "5" },
 171         { "idle_timeout",  PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, idle_timeout),
 172           0, "60" },
 173         { "spread", PW_TYPE_BOOLEAN, offsetof(fr_connection_pool_t, spread),
 174           0, "no" },
 175         { NULL, -1, 0, NULL, NULL }
 176 };
 177
 178 /** Removes a connection from the connection list
 179  *
 180  * @note Must be called with the mutex held.
 181  *
 182  * @param[in,out] pool to modify.
 183  * @param[in] this Connection to delete.
 184  */
 185 static void fr_connection_unlink(fr_connection_pool_t *pool,
 186                                  fr_connection_t *this)
 187 {
 188         if (this->prev) {
 189                 rad_assert(pool->head != this);
 190                 this->prev->next = this->next;
 191         } else {
 192                 rad_assert(pool->head == this);
 193                 pool->head = this->next;
 194         }
 195         if (this->next) {
 196                 rad_assert(pool->tail != this);
 197                 this->next->prev = this->prev;
 198         } else {
 199                 rad_assert(pool->tail == this);
 200                 pool->tail = this->prev;
 201         }
 202
 203         this->prev = this->next = NULL;
 204 }
 205
 206 /** Adds a connection to the head of the connection list
 207  *
 208  * @note Must be called with the mutex held.
 209  *
 210  * @param[in,out] pool to modify.
 211  * @param[in] this Connection to add.
 212  */
 213 static void fr_connection_link_head(fr_connection_pool_t *pool,
 214                                     fr_connection_t *this)
 215 {
 216         rad_assert(pool != NULL);
 217         rad_assert(this != NULL);
 218         rad_assert(pool->head != this);
 219         rad_assert(pool->tail != this);
 220
 221         if (pool->head) {
 222                 pool->head->prev = this;
 223         }
 224
 225         this->next = pool->head;
 226         this->prev = NULL;
 227         pool->head = this;
 228         if (!pool->tail) {
 229                 rad_assert(this->next == NULL);
 230                 pool->tail = this;
 231         } else {
 232                 rad_assert(this->next != NULL);
 233         }
 234 }
 235
 236 /** Adds a connection to the tail of the connection list
 237  *
 238  * @note Must be called with the mutex held.
 239  *
 240  * @param[in,out] pool to modify.
 241  * @param[in] this Connection to add.
 242  */
 243 static void fr_connection_link_tail(fr_connection_pool_t *pool,
 244                                     fr_connection_t *this)
 245 {
 246         rad_assert(pool != NULL);
 247         rad_assert(this != NULL);
 248         rad_assert(pool->head != this);
 249         rad_assert(pool->tail != this);
 250
 251         if (pool->tail) {
 252                 pool->tail->next = this;
 253         }
 254         this->prev = pool->tail;
 255         this->next = NULL;
 256         pool->tail = this;
 257         if (!pool->head) {
 258                 rad_assert(this->prev == NULL);
 259                 pool->head = this;
 260         } else {
 261                 rad_assert(this->prev != NULL);
 262         }
 263 }
 264
 265
 266 /** Spawns a new connection
 267  *
 268  * Spawns a new connection using the create callback, and returns it for
 269  * adding to the connection list.
 270  *
 271  * @note Will call the 'open' trigger.
 272  * @note Must be called with the mutex free.
 273  *
 274  * @param[in] pool to modify.
 275  * @param[in] now Current time.
 276  * @return the new connection struct or NULL on error.
 277  */
 278 static fr_connection_t *fr_connection_spawn(fr_connection_pool_t *pool,
 279                                             time_t now)
 280 {
 281         fr_connection_t *this;
 282         void *conn;
 283
 284         rad_assert(pool != NULL);
 285
 286         /*
 287          *      Prevent all threads from blocking if the resource
 288          *      were managing connections for appears to be unavailable.
 289          */
 290         if ((pool->num == 0) && pool->spawning) {
 291                 return NULL;
 292         }
 293
 294         pthread_mutex_lock(&pool->mutex);
 295         rad_assert(pool->num <= pool->max);
 296
 297         if ((pool->last_failed == now) || pool->spawning) {
 298                 bool complain = false;
 299
 300                 if (pool->last_throttled != now) {
 301                         complain = true;
 302
 303                         pool->last_throttled = now;
 304                 }
 305
 306                 pthread_mutex_unlock(&pool->mutex);
 307
 308                 if (complain) {
 309                         if (pool->spawning) {
 310                                 ERROR("%s: Cannot open new connection, "
 311                                        "connection spawning already in "
 312                                        "progress", pool->log_prefix);
 313                         } else {
 314                                 ERROR("%s: Last connection failed, "
 315                                        "throttling connection spawn",
 316                                        pool->log_prefix);
 317                         }
 318                 }
 319
 320                 return NULL;
 321         }
 322
 323         pool->spawning = true;
 324
 325         /*
 326          *      Unlock the mutex while we try to open a new
 327          *      connection.  If there are issues with the back-end,
 328          *      opening a new connection may take a LONG time.  In
 329          *      that case, we want the other connections to continue
 330          *      to be used.
 331          */
 332         pthread_mutex_unlock(&pool->mutex);
 333
 334         INFO("%s: Opening additional connection (%" PRIu64 ")", pool->log_prefix, pool->count);
 335
 336         this = rad_malloc(sizeof(*this));
 337         memset(this, 0, sizeof(*this));
 338
 339         /*
 340          *      This may take a long time, which prevents other
 341          *      threads from releasing connections.  We don't care
 342          *      about other threads opening new connections, as we
 343          *      already have no free connections.
 344          */
 345         conn = pool->create(pool->ctx);
 346         if (!conn) {
 347                 ERROR("%s: Opening connection failed (%" PRIu64 ")", pool->log_prefix, pool->count);
 348
 349                 pool->last_failed = now;
 350                 free(this);
 351                 pool->spawning = false; /* atomic, so no lock is needed */
 352                 return NULL;
 353         }
 354
 355         this->created = now;
 356         this->connection = conn;
 357
 358         /*
 359          *      And lock the mutex again while we link the new
 360          *      connection back into the pool.
 361          */
 362         pthread_mutex_lock(&pool->mutex);
 363
 364         this->number = pool->count++;
 365         this->last_used = now;
 366         fr_connection_link_head(pool, this);
 367         pool->num++;
 368         pool->spawning = false;
 369         pool->last_spawned = time(NULL);
 370
 371         pthread_mutex_unlock(&pool->mutex);
 372
 373         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 374
 375         return this;
 376 }
 377
 378 /** Add a new connection to the pool
 379  *
 380  * If conn is not NULL will attempt to add that connection handle to the pool.
 381  * If conn is NULL will attempt to spawn a new connection using the create
 382  * callback.
 383  *
 384  * @note Will call the 'open' trigger.
 385  *
 386  * @param[in,out] pool to add connection to.
 387  * @param[in] conn to add.
 388  * @return 0 if the connection wasn't added else 1.
 389  */
 390 int fr_connection_add(fr_connection_pool_t *pool, void *conn)
 391 {
 392         fr_connection_t *this;
 393
 394         if (!pool) return 0;
 395
 396         pthread_mutex_lock(&pool->mutex);
 397
 398         if (!conn) {
 399                 conn = pool->create(pool->ctx);
 400                 if (!conn) {
 401                         pthread_mutex_unlock(&pool->mutex);
 402                         return 0;
 403                 }
 404
 405                 INFO("%s: Opening connection successful (%" PRIu64 ")", pool->log_prefix, pool->count);
 406         }
 407
 408         /*
 409          *      Too many connections: can't add it.
 410          */
 411         if (pool->num >= pool->max) {
 412                 pthread_mutex_unlock(&pool->mutex);
 413                 return 0;
 414         }
 415
 416         this = rad_malloc(sizeof(*this));
 417         memset(this, 0, sizeof(*this));
 418
 419         this->created = time(NULL);
 420         this->connection = conn;
 421
 422         this->number = pool->count++;
 423         this->last_used = time(NULL);
 424         fr_connection_link_head(pool, this);
 425         pool->num++;
 426
 427         pthread_mutex_unlock(&pool->mutex);
 428
 429         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 430
 431         return 1;
 432 }
 433
 434 /** Close an existing connection.
 435  *
 436  * Removes the connection from the list, calls the delete callback to close
 437  * the connection, then frees memory allocated to the connection.
 438  *
 439  * @note Will call the 'close' trigger.
 440  * @note Must be called with the mutex held.
 441  *
 442  * @param[in,out] pool to modify.
 443  * @param[in,out] this Connection to delete.
 444
 445  */
 446 static void fr_connection_close(fr_connection_pool_t *pool,
 447                                 fr_connection_t *this)
 448 {
 449         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
 450
 451         rad_assert(this->in_use == false);
 452
 453         fr_connection_unlink(pool, this);
 454         pool->delete(pool->ctx, this->connection);
 455         rad_assert(pool->num > 0);
 456         pool->num--;
 457         free(this);
 458 }
 459
 460 /** Find a connection handle in the connection list
 461  *
 462  * Walks over the list of connections searching for a specified connection
 463  * handle and returns the first connection that contains that pointer.
 464  *
 465  * @note Will lock mutex and only release mutex if connection handle
 466  * is not found, so will usually return will mutex held.
 467  * @note Must be called with the mutex free.
 468  *
 469  * @param[in] pool to search in.
 470  * @param[in] conn handle to search for.
 471  * @return the connection containing the specified handle, or NULL if non is
 472  * found.
 473  */
 474 static fr_connection_t *fr_connection_find(fr_connection_pool_t *pool, void *conn)
 475 {
 476         fr_connection_t *this;
 477
 478         if (!pool || !conn) return NULL;
 479
 480         pthread_mutex_lock(&pool->mutex);
 481
 482         /*
 483          *      FIXME: This loop could be avoided if we passed a 'void
 484          *      **connection' instead.  We could use "offsetof" in
 485          *      order to find top of the parent structure.
 486          */
 487         for (this = pool->head; this != NULL; this = this->next) {
 488                 if (this->connection == conn) return this;
 489         }
 490
 491         pthread_mutex_unlock(&pool->mutex);
 492         return NULL;
 493 }
 494
 495 /** Delete a connection from the connection pool.
 496  *
 497  * Resolves the connection handle to a connection, then (if found)
 498  * closes, unlinks and frees that connection.
 499  *
 500  * @note Must be called with the mutex free.
 501  *
 502  * @param[in,out] pool Connection pool to modify.
 503  * @param[in] conn to delete.
 504  * @return 0 if the connection could not be found, else 1.
 505  */
 506 int fr_connection_del(fr_connection_pool_t *pool, void *conn)
 507 {
 508         fr_connection_t *this;
 509
 510         this = fr_connection_find(pool, conn);
 511         if (!this) return 0;
 512
 513         /*
 514          *      If it's in use, release it.
 515          */
 516         if (this->in_use) {
 517                 rad_assert(this->in_use == true);
 518                 this->in_use = false;
 519
 520                 rad_assert(pool->active > 0);
 521                 pool->active--;
 522         }
 523
 524         INFO("%s: Deleting connection (%" PRIu64 ")", pool->log_prefix, this->number);
 525
 526         fr_connection_close(pool, this);
 527         fr_connection_pool_check(pool);
 528         return 1;
 529 }
 530
 531 /** Delete a connection pool
 532  *
 533  * Closes, unlinks and frees all connections in the connection pool, then frees
 534  * all memory used by the connection pool.
 535  *
 536  * @note Will call the 'stop' trigger.
 537  * @note Must be called with the mutex free.
 538  *
 539  * @param[in,out] pool to delete.
 540  */
 541 void fr_connection_pool_delete(fr_connection_pool_t *pool)
 542 {
 543         fr_connection_t *this, *next;
 544
 545         if (!pool) return;
 546
 547         DEBUG("%s: Removing connection pool", pool->log_prefix);
 548
 549         pthread_mutex_lock(&pool->mutex);
 550
 551         for (this = pool->head; this != NULL; this = next) {
 552                 next = this->next;
 553
 554                 INFO("%s: Closing connection (%" PRIu64 ")", pool->log_prefix, this->number);
 555
 556                 fr_connection_close(pool, this);
 557         }
 558
 559         if (pool->trigger) exec_trigger(NULL, pool->cs, "stop", true);
 560
 561         rad_assert(pool->head == NULL);
 562         rad_assert(pool->tail == NULL);
 563         rad_assert(pool->num == 0);
 564
 565         free(pool->log_prefix);
 566         free(pool);
 567 }
 568
 569 /** Create a new connection pool
 570  *
 571  * Allocates structures used by the connection pool, initialises the various
 572  * configuration options and counters, and sets the callback functions.
 573  *
 574  * Will also spawn the number of connections specified by the 'start'
 575  * configuration options.
 576  *
 577  * @note Will call the 'start' trigger.
 578  *
 579  * @param[in] parent configuration section containing a 'pool' subsection.
 580  * @param[in] ctx pointer to pass to callbacks.
 581  * @param[in] c Callback to create new connections.
 582  * @param[in] a Callback to check the status of connections.
 583  * @param[in] d Callback to delete connections.
 584  * @param[in] prefix to prepend to all log message, if NULL will create prefix
 585  *      from parent conf section names.
 586  * @return A new connection pool or NULL on error.
 587  */
 588 fr_connection_pool_t *fr_connection_pool_init(CONF_SECTION *parent,
 589                                               void *ctx,
 590                                               fr_connection_create_t c,
 591                                               fr_connection_alive_t a,
 592                                               fr_connection_delete_t d,
 593                                               char *prefix)
 594 {
 595         int i, lp_len;
 596         fr_connection_pool_t *pool;
 597         fr_connection_t *this;
 598         CONF_SECTION *modules;
 599         CONF_SECTION *cs;
 600         char const *cs_name1, *cs_name2;
 601         time_t now = time(NULL);
 602
 603         if (!parent || !ctx || !c || !d) return NULL;
 604
 605         cs = cf_section_sub_find(parent, "pool");
 606         if (!cs) cs = cf_section_sub_find(parent, "limit");
 607
 608         pool = rad_malloc(sizeof(*pool));
 609         memset(pool, 0, sizeof(*pool));
 610
 611         pool->cs = cs;
 612         pool->ctx = ctx;
 613         pool->create = c;
 614         pool->alive = a;
 615         pool->delete = d;
 616
 617         pool->head = pool->tail = NULL;
 618
 619 #ifdef HAVE_PTHREAD_H
 620         pthread_mutex_init(&pool->mutex, NULL);
 621 #endif
 622
 623         if (!prefix) {
 624                 modules = cf_item_parent(cf_sectiontoitem(parent));
 625                 if (modules) {
 626                         cs_name1 = cf_section_name1(modules);
 627                         if (cs_name1 && (strcmp(cs_name1, "modules") == 0)) {
 628                                 cs_name1 = cf_section_name1(parent);
 629                                 cs_name2 = cf_section_name2(parent);
 630                                 if (!cs_name2) {
 631                                         cs_name2 = cs_name1;
 632                                 }
 633
 634                                 lp_len = (sizeof(LOG_PREFIX) - 4) + strlen(cs_name1) + strlen(cs_name2);
 635                                 pool->log_prefix = rad_malloc(lp_len);
 636                                 snprintf(pool->log_prefix, lp_len, LOG_PREFIX, cs_name1,
 637                                          cs_name2);
 638                         }
 639                 } else {                /* not a module configuration */
 640                         cs_name1 = cf_section_name1(parent);
 641
 642                         pool->log_prefix = strdup(cs_name1);
 643                 }
 644         } else {
 645                 pool->log_prefix = strdup(prefix);
 646         }
 647
 648         DEBUG("%s: Initialising connection pool", pool->log_prefix);
 649
 650         if (cs) {
 651                 if (cf_section_parse(cs, pool, connection_config) < 0) {
 652                         goto error;
 653                 }
 654
 655                 if (cf_section_sub_find(cs, "trigger")) pool->trigger = true;
 656         } else {
 657                 pool->start = 5;
 658                 pool->min = 5;
 659                 pool->max = 10;
 660                 pool->spare = 3;
 661                 pool->cleanup_delay = 5;
 662                 pool->idle_timeout = 60;
 663         }
 664
 665         /*
 666          *      Some simple limits
 667          */
 668         if (pool->max == 0) {
 669                 cf_log_err_cs(cs, "Cannot set 'max' to zero");
 670                 goto error;
 671         }
 672         if (pool->min > pool->max) {
 673                 cf_log_err_cs(cs, "Cannot set 'min' to more than 'max'");
 674                 goto error;
 675         }
 676
 677         if (pool->max > 1024) pool->max = 1024;
 678         if (pool->start > pool->max) pool->start = pool->max;
 679         if (pool->spare > (pool->max - pool->min)) {
 680                 pool->spare = pool->max - pool->min;
 681         }
 682         if ((pool->lifetime > 0) && (pool->idle_timeout > pool->lifetime)) {
 683                 pool->idle_timeout = 0;
 684         }
 685
 686         /*
 687          *      Create all of the connections, unless the admin says
 688          *      not to.
 689          */
 690         for (i = 0; i < pool->start; i++) {
 691                 this = fr_connection_spawn(pool, now);
 692                 if (!this) {
 693                 error:
 694                         fr_connection_pool_delete(pool);
 695                         return NULL;
 696                 }
 697         }
 698
 699         if (pool->trigger) exec_trigger(NULL, pool->cs, "start", true);
 700
 701         return pool;
 702 }
 703
 704
 705 /** Check whether a connection needs to be removed from the pool
 706  *
 707  * Will verify that the connection is within idle_timeout, max_uses, and
 708  * lifetime values. If it is not, the connection will be closed.
 709  *
 710  * @note Will only close connections not in use.
 711  * @note Must be called with the mutex held.
 712  *
 713  * @param[in,out] pool to modify.
 714  * @param[in,out] this Connection to manage.
 715  * @param[in] now Current time.
 716  * @return 0 if the connection was closed, otherwise 1.
 717  */
 718 static int fr_connection_manage(fr_connection_pool_t *pool,
 719                                 fr_connection_t *this,
 720                                 time_t now)
 721 {
 722         rad_assert(pool != NULL);
 723         rad_assert(this != NULL);
 724
 725         /*
 726          *      Don't terminated in-use connections
 727          */
 728         if (this->in_use) return 1;
 729
 730         if ((pool->max_uses > 0) &&
 731             (this->num_uses >= pool->max_uses)) {
 732                 DEBUG("%s: Closing expired connection (%" PRIu64 "): Hit max_uses limit", pool->log_prefix,
 733                       this->number);
 734         do_delete:
 735                 if ((pool->num <= pool->min) &&
 736                     (pool->last_complained < now)) {
 737                         WARN("%s: You probably need to lower \"min\"", pool->log_prefix);
 738
 739                         pool->last_complained = now;
 740                 }
 741                 fr_connection_close(pool, this);
 742                 return 0;
 743         }
 744
 745         if ((pool->lifetime > 0) &&
 746             ((this->created + pool->lifetime) < now)) {
 747                 DEBUG("%s: Closing expired connection (%" PRIu64 ")", pool->log_prefix, this->number);
 748                 goto do_delete;
 749         }
 750
 751         if ((pool->idle_timeout > 0) &&
 752             ((this->last_used + pool->idle_timeout) < now)) {
 753                 INFO("%s: Closing connection (%" PRIu64 "): Hit idle_timeout, was idle for %u seconds",
 754                      pool->log_prefix, this->number, (int) (now - this->last_used));
 755                 goto do_delete;
 756         }
 757
 758         return 1;
 759 }
 760
 761
 762 /** Check whether any connections needs to be removed from the pool
 763  *
 764  * Maintains the number of connections in the pool as per the configuration
 765  * parameters for the connection pool.
 766  *
 767  * @note Will only run checks the first time it's called in a given second,
 768  * to throttle connection spawning/closing.
 769  * @note Will only close connections not in use.
 770  * @note Must be called with the mutex held, will release mutex before
 771  * returning.
 772  *
 773  * @param[in,out] pool to manage.
 774  * @return 1
 775  */
 776 static int fr_connection_pool_check(fr_connection_pool_t *pool)
 777 {
 778         int spare, spawn;
 779         time_t now = time(NULL);
 780         fr_connection_t *this, *next;
 781
 782         if (pool->last_checked == now) {
 783                 pthread_mutex_unlock(&pool->mutex);
 784                 return 1;
 785         }
 786
 787         spare = pool->num - pool->active;
 788
 789         if ((pool->num < pool->max) && (spare < pool->spare)) {
 790                 spawn = pool->spare - spare;
 791                 if ((spawn + pool->num) > pool->max) {
 792                         spawn = pool->max - pool->num;
 793                 }
 794                 if (pool->spawning) spawn = 0;
 795
 796                 if (spawn) {
 797                         pthread_mutex_unlock(&pool->mutex);
 798                         fr_connection_spawn(pool, now); /* ignore return code */
 799                         pthread_mutex_lock(&pool->mutex);
 800                 }
 801         }
 802
 803         /*
 804          *      We haven't spawned connections in a while, and there
 805          *      are too many spare ones.  Close the one which has been
 806          *      idle for the longest.
 807          */
 808         if ((now >= (pool->last_spawned + pool->cleanup_delay)) &&
 809             (spare > pool->spare)) {
 810                 fr_connection_t *idle;
 811
 812                 idle = NULL;
 813                 for (this = pool->tail; this != NULL; this = this->prev) {
 814                         if (this->in_use) continue;
 815
 816                         if (!idle ||
 817                            (this->last_used < idle->last_used)) {
 818                                 idle = this;
 819                         }
 820                 }
 821
 822                 rad_assert(idle != NULL);
 823
 824                 INFO("%s: Closing connection (%" PRIu64 "): Too many free connections (%d > %d)", pool->log_prefix,
 825                      idle->number, spare, pool->spare);
 826                 fr_connection_close(pool, idle);
 827         }
 828
 829         /*
 830          *      Pass over all of the connections in the pool, limiting
 831          *      lifetime, idle time, max requests, etc.
 832          */
 833         for (this = pool->head; this != NULL; this = next) {
 834                 next = this->next;
 835                 fr_connection_manage(pool, this, now);
 836         }
 837
 838         pool->last_checked = now;
 839         pthread_mutex_unlock(&pool->mutex);
 840
 841         return 1;
 842 }
 843
 844 /** Trigger connection check for a given connection or all connections
 845  *
 846  * If conn is not NULL then we call fr_connection_manage on the connection.
 847  * If conn is NULL we call fr_connection_pool_check on the pool.
 848  *
 849  * @note Only connections that are not in use will be closed.
 850  *
 851  * @see fr_connection_manage
 852  * @see fr_connection_pool_check
 853  * @param[in,out] pool to manage.
 854  * @param[in,out] conn to check.
 855  * @return 0 if the connection was closed, else 1.
 856  */
 857 int fr_connection_check(fr_connection_pool_t *pool, void *conn)
 858 {
 859         fr_connection_t *this;
 860         time_t now;
 861         int ret = 1;
 862
 863         if (!pool) return 1;
 864
 865         now = time(NULL);
 866         pthread_mutex_lock(&pool->mutex);
 867
 868         if (!conn) return fr_connection_pool_check(pool);
 869
 870         for (this = pool->head; this != NULL; this = this->next) {
 871                 if (this->connection == conn) {
 872                         ret = fr_connection_manage(pool, conn, now);
 873                         break;
 874                 }
 875         }
 876
 877         pthread_mutex_unlock(&pool->mutex);
 878
 879         return ret;
 880 }
 881
 882 /** Reserve a connection in the connection pool
 883  *
 884  * Will attempt to find an unused connection in the connection pool, if one is
 885  * found, will mark it as in in use increment the number of active connections
 886  * and return the connection handle.
 887  *
 888  * If no free connections are found will attempt to spawn a new one, conditional
 889  * on a connection spawning not already being in progress, and not being at the
 890  * 'max' connection limit.
 891  *
 892  * @note fr_connection_release must be called once the caller has finished
 893  * using the connection.
 894  *
 895  * @see fr_connection_release
 896  * @param[in,out] pool to reserve the connection from.
 897  * @return a pointer to the connection handle, or NULL on error.
 898  */
 899 void *fr_connection_get(fr_connection_pool_t *pool)
 900 {
 901         time_t now;
 902         fr_connection_t *this, *next;
 903
 904         if (!pool) return NULL;
 905
 906         pthread_mutex_lock(&pool->mutex);
 907
 908         now = time(NULL);
 909         for (this = pool->head; this != NULL; this = next) {
 910                 next = this->next;
 911
 912                 if (!this->in_use) goto do_return;
 913         }
 914
 915         if (pool->num == pool->max) {
 916                 bool complain = false;
 917
 918                 /*
 919                  *      Rate-limit complaints.
 920                  */
 921                 if (pool->last_at_max != now) {
 922                         complain = true;
 923                         pool->last_at_max = now;
 924                 }
 925
 926                 pthread_mutex_unlock(&pool->mutex);
 927
 928                 if (complain) {
 929                         ERROR("%s: No connections available and at max connection limit", pool->log_prefix);
 930                 }
 931
 932                 return NULL;
 933         }
 934
 935         pthread_mutex_unlock(&pool->mutex);
 936         this = fr_connection_spawn(pool, now);
 937         if (!this) return NULL;
 938         pthread_mutex_lock(&pool->mutex);
 939
 940 do_return:
 941         pool->active++;
 942         this->num_uses++;
 943         this->last_used = now;
 944         this->in_use = true;
 945
 946         pthread_mutex_unlock(&pool->mutex);
 947
 948         DEBUG("%s: Reserved connection (%" PRIu64 ")", pool->log_prefix, this->number);
 949
 950         return this->connection;
 951 }
 952
 953 /** Release a connection
 954  *
 955  * Will mark a connection as unused and decrement the number of active
 956  * connections.
 957  *
 958  * @see fr_connection_get
 959  * @param[in,out] pool to release the connection in.
 960  * @param[in,out] conn to release.
 961  */
 962 void fr_connection_release(fr_connection_pool_t *pool, void *conn)
 963 {
 964         fr_connection_t *this;
 965
 966         this = fr_connection_find(pool, conn);
 967         if (!this) return;
 968
 969         rad_assert(this->in_use == true);
 970         this->in_use = false;
 971
 972         /*
 973          *      Determines whether the last used connection gets
 974          *      re-used first.
 975          */
 976         if (pool->spread) {
 977                 /*
 978                  *      Put it at the tail of the list, so
 979                  *      that it will get re-used last.
 980                  */
 981                 if (this != pool->tail) {
 982                         fr_connection_unlink(pool, this);
 983                         fr_connection_link_tail(pool, this);
 984                 }
 985         } else {
 986                 /*
 987                  *      Put it at the head of the list, so
 988                  *      that it will get re-used quickly.
 989                  */
 990                 if (this != pool->head) {
 991                         fr_connection_unlink(pool, this);
 992                         fr_connection_link_head(pool, this);
 993                 }
 994         }
 995
 996         rad_assert(pool->active > 0);
 997         pool->active--;
 998
 999         DEBUG("%s: Released connection (%" PRIu64 ")", pool->log_prefix, this->number);
1000
1001         /*
1002          *      We mirror the "spawn on get" functionality by having
1003          *      "delete on release".  If there are too many spare
1004          *      connections, go manage the pool && clean some up.
1005          */
1006         fr_connection_pool_check(pool);
1007 }
1008
1009 /** Reconnect a suspected inviable connection
1010  *
1011  * This should be called by the module if it suspects that a connection is
1012  * not viable (e.g. the server has closed it).
1013  *
1014  * Will attempt to create a new connection handle using the create callback,
1015  * and if this is successful the new handle will be assigned to the existing
1016  * pool connection.
1017  *
1018  * If this is not successful, the connection will be removed from the pool.
1019  *
1020  * When implementing a module that uses the connection pool API, it is advisable
1021  * to pass a pointer to the pointer to the handle (void **conn)
1022  * to all functions which may call reconnect. This is so that if a new handle
1023  * is created and returned, the handle pointer can be updated up the callstack,
1024  * and a function higher up the stack doesn't attempt to use a now invalid
1025  * connection handle.
1026  *
1027  * @warning After calling reconnect the caller *MUST NOT* attempt to use
1028  * the old handle in any other operations, as its memory will have been freed.
1029  *
1030  * @see fr_connection_get
1031  * @param[in,out] pool to reconnect the connection in.
1032  * @param[in,out] conn to reconnect.
1033  * @return ew connection handle if successful else NULL.
1034  */
1035 void *fr_connection_reconnect(fr_connection_pool_t *pool, void *conn)
1036 {
1037         void *new_conn;
1038         fr_connection_t *this;
1039         uint64_t conn_number;
1040
1041         if (!pool || !conn) return NULL;
1042
1043         this = fr_connection_find(pool, conn);
1044         if (!this) return NULL;
1045
1046         /*
1047          *      The pool is now locked.
1048          */
1049         conn_number = this->number;
1050
1051         DEBUG("%s: Reconnecting (%" PRIu64 ")", pool->log_prefix, conn_number);
1052
1053         new_conn = pool->create(pool->ctx);
1054         if (!new_conn) {
1055                 time_t now = time(NULL);
1056
1057                 if (pool->last_complained == now) {
1058                         now = 0;
1059                 } else {
1060                         pool->last_complained = now;
1061                 }
1062
1063                 /*
1064                  *      The caller has said the connection is bad.  So
1065                  *      if it's still in use, mark it as unused, and
1066                  *      close it.  If it's not in use, we just try to
1067                  *      get a new connection.
1068                  */
1069                 if (this->in_use) {
1070                         this->in_use = false;
1071
1072                         rad_assert(pool->active > 0);
1073                         pool->active--;
1074
1075                         fr_connection_close(pool, this);
1076                 }
1077
1078                 /*
1079                  *      We failed to create a new socket.
1080                  *      Try to grab an existing one.
1081                  */
1082                 pthread_mutex_unlock(&pool->mutex);
1083                 new_conn = fr_connection_get(pool);
1084                 if (new_conn) return new_conn;
1085
1086                 if (!now) return NULL;
1087
1088                 ERROR("%s: Failed to reconnect (%" PRIu64 "), and no other connections available", pool->log_prefix,
1089                       conn_number);
1090
1091                 return NULL;
1092         }
1093
1094         pool->delete(pool->ctx, conn);
1095         this->connection = new_conn;
1096         pthread_mutex_unlock(&pool->mutex);
1097         return new_conn;
1098 }