src/main/connection.c

   1 /*
   2  *   This program is free software; you can redistribute it and/or modify
   3  *   it under the terms of the GNU General Public License as published by
   4  *   the Free Software Foundation; either version 2 of the License, or
   5  *   (at your option) any later version.
   6  *
   7  *   This program is distributed in the hope that it will be useful,
   8  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  *   GNU General Public License for more details.
  11  *
  12  *   You should have received a copy of the GNU General Public License
  13  *   along with this program; if not, write to the Free Software
  14  *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  15  */
  16
  17 /**
  18  * @file connection.c
  19  * @brief Handle pools of connections (threads, sockets, etc.)
  20  * @note This API must be used by all modules in the public distribution that
  21  * maintain pools of connections.
  22  *
  23  * @copyright 2012  The FreeRADIUS server project
  24  * @copyright 2012  Alan DeKok <aland@deployingradius.com>
  25  */
  26 RCSID("$Id$")
  27
  28 #include <freeradius-devel/radiusd.h>
  29
  30 #include <freeradius-devel/connection.h>
  31
  32 #include <freeradius-devel/rad_assert.h>
  33
  34 typedef struct fr_connection fr_connection_t;
  35
  36 static int fr_connection_pool_check(fr_connection_pool_t *pool);
  37
  38 /** An individual connection within the connection pool
  39  *
  40  * Defines connection counters, timestamps, and holds a pointer to the
  41  * connection handle itself.
  42  *
  43  * @see fr_connection_pool_t
  44  */
  45 struct fr_connection {
  46         fr_connection_t *prev;          //!< Previous connection in list.
  47         fr_connection_t *next;          //!< Next connection in list.
  48
  49         time_t          created;        //!< Time connection was created.
  50         time_t          last_used;      //!< Last time the connection was
  51                                         //!< reserved.
  52
  53         uint64_t        num_uses;       //!< Number of times the connection
  54                                         //!< has been reserved.
  55         int             in_use;         //!< Whether the connection is currently
  56                                         //!< reserved.
  57         uint64_t        number;         //!< Unique ID assigned when the
  58                                         //!< connection is created, these will
  59                                         //!< monotonically increase over the
  60                                         //!< lifetime of the connection pool.
  61         void            *connection;    //!< Pointer to whatever the module
  62                                         //!< uses for a connection handle.
  63 };
  64
  65 /** A connection pool
  66  *
  67  * Defines the configuration of the connection pool, all the counters and
  68  * timestamps related to the connection pool, the mutex that stops multiple
  69  * threads leaving the pool in an inconsistent state, and the callbacks
  70  * required to open, close and check the status of connections within the pool.
  71  *
  72  * @see fr_connection
  73  */
  74 struct fr_connection_pool_t {
  75         int             start;          //!< Number of initial connections
  76         int             min;            //!< Minimum number of concurrent
  77                                         //!< connections to keep open.
  78         int             max;            //!< Maximum number of concurrent
  79                                         //!< connections to allow.
  80         int             spare;          //!< Number of spare connections to try
  81                                         //!< and maintain.
  82         int             cleanup_delay;  //!< How long a connection can go unused
  83                                         //!< for before it's closed
  84                                         //!< (0 is infinite).
  85         uint64_t        max_uses;       //!< Maximum number of times a
  86                                         //!< connection can be used before being
  87                                         //!< closed.
  88         int             lifetime;       //!< How long a connection can be open
  89                                         //!< before being closed (irrespective
  90                                         //!< of whether it's idle or not).
  91         int             idle_timeout;   //!< How long a connection can be idle
  92                                         //!< before being closed.
  93
  94         int             trigger;        //!< If true execute connection triggers
  95                                         //!< associated with the connection
  96                                         //!< pool.
  97
  98         int             spread;         //!< If true requests will be spread
  99                                         //!< across all connections, instead of
 100                                         //!< re-using the most recently used
 101                                         //!< connections first.
 102
 103         time_t          last_checked;   //!< Last time we pruned the connection
 104                                         //!< pool.
 105         time_t          last_spawned;   //!< Last time we spawned a connection.
 106         time_t          last_failed;    //!< Last time we tried to spawn a
 107                                         //!< a connection but failed.
 108         time_t          last_complained;//!< Last time we complained about
 109                                         //!< configuration parameters.
 110         time_t          last_throttled; //!< Last time we refused to spawn a
 111                                         //!< connection because the last
 112                                         //!< connection failed, or we were
 113                                         //!< already spawning a connection.
 114         time_t          last_at_max;    //!< Last time we hit the maximum number
 115                                         //!< of allowed connections.
 116
 117         uint64_t        count;          //!< Number of connections spawned over
 118                                         //!< the lifetime of the pool.
 119         int             num;            //!< Number of connections in the pool.
 120         int             active;         //!< Number of currently reserved
 121                                         //!< connections.
 122
 123         fr_connection_t *head;          //!< Start of the connection list.
 124         fr_connection_t *tail;          //!< End of the connection list.
 125
 126         bool            spawning;       //!< Whether we are currently attempting
 127                                         //!< to spawn a new connection.
 128
 129 #ifdef HAVE_PTHREAD_H
 130         pthread_mutex_t mutex;          //!< Mutex used to keep consistent state
 131                                         //!< when making modifications in
 132                                         //!< threaded mode.
 133 #endif
 134
 135         CONF_SECTION    *cs;            //!< Configuration section holding
 136                                         //!< the section of parsed config file
 137                                         //!< that relates to this pool.
 138         void            *ctx;           //!< Pointer to context data that will
 139                                         //!< be passed to callbacks.
 140
 141         char            *log_prefix;    //!< Log prefix to prepend to all log
 142                                         //!< messages created by the connection
 143                                         //!< pool code.
 144
 145         fr_connection_create_t  create; //!< Function used to create new
 146                                         //!< connections.
 147         fr_connection_alive_t   alive;  //!< Function used to check status
 148                                         //!< of connections.
 149         fr_connection_delete_t  delete; //!< Function used to close existing
 150                                         //!< connections.
 151 };
 152
 153 #define LOG_PREFIX "rlm_%s (%s)"
 154 #ifndef HAVE_PTHREAD_H
 155 #define pthread_mutex_lock(_x)
 156 #define pthread_mutex_unlock(_x)
 157 #endif
 158
 159 static const CONF_PARSER connection_config[] = {
 160         { "start",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, start),
 161           0, "5" },
 162         { "min",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, min),
 163           0, "5" },
 164         { "max",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max),
 165           0, "10" },
 166         { "spare",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, spare),
 167           0, "3" },
 168         { "uses",     PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max_uses),
 169           0, "0" },
 170         { "lifetime", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, lifetime),
 171           0, "0" },
 172         { "cleanup_delay", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, cleanup_delay),
 173           0, "5" },
 174         { "idle_timeout",  PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, idle_timeout),
 175           0, "60" },
 176         { "spread", PW_TYPE_BOOLEAN, offsetof(fr_connection_pool_t, spread),
 177           0, "no" },
 178         { NULL, -1, 0, NULL, NULL }
 179 };
 180
 181 /** Removes a connection from the connection list
 182  *
 183  * @note Must be called with the mutex held.
 184  *
 185  * @param[in,out] pool to modify.
 186  * @param[in] this Connection to delete.
 187  */
 188 static void fr_connection_unlink(fr_connection_pool_t *pool,
 189                                  fr_connection_t *this)
 190 {
 191         if (this->prev) {
 192                 rad_assert(pool->head != this);
 193                 this->prev->next = this->next;
 194         } else {
 195                 rad_assert(pool->head == this);
 196                 pool->head = this->next;
 197         }
 198         if (this->next) {
 199                 rad_assert(pool->tail != this);
 200                 this->next->prev = this->prev;
 201         } else {
 202                 rad_assert(pool->tail == this);
 203                 pool->tail = this->prev;
 204         }
 205
 206         this->prev = this->next = NULL;
 207 }
 208
 209 /** Adds a connection to the head of the connection list
 210  *
 211  * @note Must be called with the mutex held.
 212  *
 213  * @param[in,out] pool to modify.
 214  * @param[in] this Connection to add.
 215  */
 216 static void fr_connection_link_head(fr_connection_pool_t *pool,
 217                                     fr_connection_t *this)
 218 {
 219         rad_assert(pool != NULL);
 220         rad_assert(this != NULL);
 221         rad_assert(pool->head != this);
 222         rad_assert(pool->tail != this);
 223
 224         if (pool->head) {
 225                 pool->head->prev = this;
 226         }
 227
 228         this->next = pool->head;
 229         this->prev = NULL;
 230         pool->head = this;
 231         if (!pool->tail) {
 232                 rad_assert(this->next == NULL);
 233                 pool->tail = this;
 234         } else {
 235                 rad_assert(this->next != NULL);
 236         }
 237 }
 238
 239 /** Adds a connection to the tail of the connection list
 240  *
 241  * @note Must be called with the mutex held.
 242  *
 243  * @param[in,out] pool to modify.
 244  * @param[in] this Connection to add.
 245  */
 246 static void fr_connection_link_tail(fr_connection_pool_t *pool,
 247                                     fr_connection_t *this)
 248 {
 249         rad_assert(pool != NULL);
 250         rad_assert(this != NULL);
 251         rad_assert(pool->head != this);
 252         rad_assert(pool->tail != this);
 253
 254         if (pool->tail) {
 255                 pool->tail->next = this;
 256         }
 257         this->prev = pool->tail;
 258         this->next = NULL;
 259         pool->tail = this;
 260         if (!pool->head) {
 261                 rad_assert(this->prev == NULL);
 262                 pool->head = this;
 263         } else {
 264                 rad_assert(this->prev != NULL);
 265         }
 266 }
 267
 268
 269 /** Spawns a new connection
 270  *
 271  * Spawns a new connection using the create callback, and returns it for
 272  * adding to the connection list.
 273  *
 274  * @note Will call the 'open' trigger.
 275  * @note Must be called with the mutex free.
 276  *
 277  * @param[in] pool to modify.
 278  * @param[in] now Current time.
 279  * @return the new connection struct or NULL on error.
 280  */
 281 static fr_connection_t *fr_connection_spawn(fr_connection_pool_t *pool,
 282                                             time_t now)
 283 {
 284         fr_connection_t *this;
 285         void *conn;
 286
 287         rad_assert(pool != NULL);
 288
 289         /*
 290          *      Prevent all threads from blocking if the resource
 291          *      were managing connections for appears to be unavailable.
 292          */
 293         if ((pool->num == 0) && pool->spawning) {
 294                 return NULL;
 295         }
 296
 297         pthread_mutex_lock(&pool->mutex);
 298         rad_assert(pool->num <= pool->max);
 299
 300         if ((pool->last_failed == now) || pool->spawning) {
 301                 int complain = false;
 302
 303                 if (pool->last_throttled != now) {
 304                         complain = true;
 305
 306                         pool->last_throttled = now;
 307                 }
 308
 309                 pthread_mutex_unlock(&pool->mutex);
 310
 311                 if (complain) {
 312                         if (pool->spawning) {
 313                                 ERROR("%s: Cannot open new connection, "
 314                                        "connection spawning already in "
 315                                        "progress", pool->log_prefix);
 316                         } else {
 317                                 ERROR("%s: Last connection failed, "
 318                                        "throttling connection spawn",
 319                                        pool->log_prefix);
 320                         }
 321                 }
 322
 323                 return NULL;
 324         }
 325
 326         pool->spawning = true;
 327
 328         /*
 329          *      Unlock the mutex while we try to open a new
 330          *      connection.  If there are issues with the back-end,
 331          *      opening a new connection may take a LONG time.  In
 332          *      that case, we want the other connections to continue
 333          *      to be used.
 334          */
 335         pthread_mutex_unlock(&pool->mutex);
 336
 337         INFO("%s: Opening additional connection (%" PRIu64 ")", pool->log_prefix, pool->count);
 338
 339         this = rad_malloc(sizeof(*this));
 340         memset(this, 0, sizeof(*this));
 341
 342         /*
 343          *      This may take a long time, which prevents other
 344          *      threads from releasing connections.  We don't care
 345          *      about other threads opening new connections, as we
 346          *      already have no free connections.
 347          */
 348         conn = pool->create(pool->ctx);
 349         if (!conn) {
 350                 ERROR("%s: Opening connection failed (%" PRIu64 ")", pool->log_prefix, pool->count);
 351
 352                 pool->last_failed = now;
 353                 free(this);
 354                 pool->spawning = false; /* atomic, so no lock is needed */
 355                 return NULL;
 356         }
 357
 358         this->created = now;
 359         this->connection = conn;
 360
 361         /*
 362          *      And lock the mutex again while we link the new
 363          *      connection back into the pool.
 364          */
 365         pthread_mutex_lock(&pool->mutex);
 366
 367         this->number = pool->count++;
 368         this->last_used = now;
 369         fr_connection_link_head(pool, this);
 370         pool->num++;
 371         pool->spawning = false;
 372         pool->last_spawned = time(NULL);
 373
 374         pthread_mutex_unlock(&pool->mutex);
 375
 376         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 377
 378         return this;
 379 }
 380
 381 /** Add a new connection to the pool
 382  *
 383  * If conn is not NULL will attempt to add that connection handle to the pool.
 384  * If conn is NULL will attempt to spawn a new connection using the create
 385  * callback.
 386  *
 387  * @note Will call the 'open' trigger.
 388  *
 389  * @param[in,out] pool to add connection to.
 390  * @param[in] conn to add.
 391  * @return 0 if the connection wasn't added else 1.
 392  */
 393 int fr_connection_add(fr_connection_pool_t *pool, void *conn)
 394 {
 395         fr_connection_t *this;
 396
 397         if (!pool) return 0;
 398
 399         pthread_mutex_lock(&pool->mutex);
 400
 401         if (!conn) {
 402                 conn = pool->create(pool->ctx);
 403                 if (!conn) {
 404                         pthread_mutex_unlock(&pool->mutex);
 405                         return 0;
 406                 }
 407
 408                 INFO("%s: Opening connection successful (%" PRIu64 ")", pool->log_prefix, pool->count);
 409         }
 410
 411         /*
 412          *      Too many connections: can't add it.
 413          */
 414         if (pool->num >= pool->max) {
 415                 pthread_mutex_unlock(&pool->mutex);
 416                 return 0;
 417         }
 418
 419         this = rad_malloc(sizeof(*this));
 420         memset(this, 0, sizeof(*this));
 421
 422         this->created = time(NULL);
 423         this->connection = conn;
 424
 425         this->number = pool->count++;
 426         this->last_used = time(NULL);
 427         fr_connection_link_head(pool, this);
 428         pool->num++;
 429
 430         pthread_mutex_unlock(&pool->mutex);
 431
 432         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 433
 434         return 1;
 435 }
 436
 437 /** Close an existing connection.
 438  *
 439  * Removes the connection from the list, calls the delete callback to close
 440  * the connection, then frees memory allocated to the connection.
 441  *
 442  * @note Will call the 'close' trigger.
 443  * @note Must be called with the mutex held.
 444  *
 445  * @param[in,out] pool to modify.
 446  * @param[in,out] this Connection to delete.
 447
 448  */
 449 static void fr_connection_close(fr_connection_pool_t *pool,
 450                                 fr_connection_t *this)
 451 {
 452         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
 453
 454         rad_assert(this->in_use == false);
 455
 456         fr_connection_unlink(pool, this);
 457         pool->delete(pool->ctx, this->connection);
 458         rad_assert(pool->num > 0);
 459         pool->num--;
 460         free(this);
 461 }
 462
 463 /** Find a connection handle in the connection list
 464  *
 465  * Walks over the list of connections searching for a specified connection
 466  * handle and returns the first connection that contains that pointer.
 467  *
 468  * @note Will lock mutex and only release mutex if connection handle
 469  * is not found, so will usually return will mutex held.
 470  * @note Must be called with the mutex free.
 471  *
 472  * @param[in] pool to search in.
 473  * @param[in] conn handle to search for.
 474  * @return the connection containing the specified handle, or NULL if non is
 475  * found.
 476  */
 477 static fr_connection_t *fr_connection_find(fr_connection_pool_t *pool, void *conn)
 478 {
 479         fr_connection_t *this;
 480
 481         if (!pool || !conn) return NULL;
 482
 483         pthread_mutex_lock(&pool->mutex);
 484
 485         /*
 486          *      FIXME: This loop could be avoided if we passed a 'void
 487          *      **connection' instead.  We could use "offsetof" in
 488          *      order to find top of the parent structure.
 489          */
 490         for (this = pool->head; this != NULL; this = this->next) {
 491                 if (this->connection == conn) return this;
 492         }
 493
 494         pthread_mutex_unlock(&pool->mutex);
 495         return NULL;
 496 }
 497
 498 /** Delete a connection from the connection pool.
 499  *
 500  * Resolves the connection handle to a connection, then (if found)
 501  * closes, unlinks and frees that connection.
 502  *
 503  * @note Must be called with the mutex free.
 504  *
 505  * @param[in,out] pool Connection pool to modify.
 506  * @param[in] conn to delete.
 507  * @return 0 if the connection could not be found, else 1.
 508  */
 509 int fr_connection_del(fr_connection_pool_t *pool, void *conn)
 510 {
 511         fr_connection_t *this;
 512
 513         this = fr_connection_find(pool, conn);
 514         if (!this) return 0;
 515
 516         /*
 517          *      If it's in use, release it.
 518          */
 519         if (this->in_use) {
 520                 rad_assert(this->in_use == true);
 521                 this->in_use = false;
 522
 523                 rad_assert(pool->active > 0);
 524                 pool->active--;
 525         }
 526
 527         INFO("%s: Deleting connection (%" PRIu64 ")", pool->log_prefix, this->number);
 528
 529         fr_connection_close(pool, this);
 530         fr_connection_pool_check(pool);
 531         return 1;
 532 }
 533
 534 /** Delete a connection pool
 535  *
 536  * Closes, unlinks and frees all connections in the connection pool, then frees
 537  * all memory used by the connection pool.
 538  *
 539  * @note Will call the 'stop' trigger.
 540  * @note Must be called with the mutex free.
 541  *
 542  * @param[in,out] pool to delete.
 543  */
 544 void fr_connection_pool_delete(fr_connection_pool_t *pool)
 545 {
 546         fr_connection_t *this, *next;
 547
 548         if (!pool) return;
 549
 550         DEBUG("%s: Removing connection pool", pool->log_prefix);
 551
 552         pthread_mutex_lock(&pool->mutex);
 553
 554         for (this = pool->head; this != NULL; this = next) {
 555                 next = this->next;
 556
 557                 INFO("%s: Closing connection (%" PRIu64 ")", pool->log_prefix, this->number);
 558
 559                 fr_connection_close(pool, this);
 560         }
 561
 562         if (pool->trigger) exec_trigger(NULL, pool->cs, "stop", true);
 563
 564         rad_assert(pool->head == NULL);
 565         rad_assert(pool->tail == NULL);
 566         rad_assert(pool->num == 0);
 567
 568         free(pool->log_prefix);
 569         free(pool);
 570 }
 571
 572 /** Create a new connection pool
 573  *
 574  * Allocates structures used by the connection pool, initialises the various
 575  * configuration options and counters, and sets the callback functions.
 576  *
 577  * Will also spawn the number of connections specified by the 'start'
 578  * configuration options.
 579  *
 580  * @note Will call the 'start' trigger.
 581  *
 582  * @param[in] parent configuration section containing a 'pool' subsection.
 583  * @param[in] ctx pointer to pass to callbacks.
 584  * @param[in] c Callback to create new connections.
 585  * @param[in] a Callback to check the status of connections.
 586  * @param[in] d Callback to delete connections.
 587  * @param[in] prefix to prepend to all log message, if NULL will create prefix
 588  *      from parent conf section names.
 589  * @return A new connection pool or NULL on error.
 590  */
 591 fr_connection_pool_t *fr_connection_pool_init(CONF_SECTION *parent,
 592                                               void *ctx,
 593                                               fr_connection_create_t c,
 594                                               fr_connection_alive_t a,
 595                                               fr_connection_delete_t d,
 596                                               char *prefix)
 597 {
 598         int i, lp_len;
 599         fr_connection_pool_t *pool;
 600         fr_connection_t *this;
 601         CONF_SECTION *modules;
 602         CONF_SECTION *cs;
 603         char const *cs_name1, *cs_name2;
 604         time_t now = time(NULL);
 605
 606         if (!parent || !ctx || !c || !d) return NULL;
 607
 608         cs = cf_section_sub_find(parent, "pool");
 609         if (!cs) cs = cf_section_sub_find(parent, "limit");
 610
 611         pool = rad_malloc(sizeof(*pool));
 612         memset(pool, 0, sizeof(*pool));
 613
 614         pool->cs = cs;
 615         pool->ctx = ctx;
 616         pool->create = c;
 617         pool->alive = a;
 618         pool->delete = d;
 619
 620         pool->head = pool->tail = NULL;
 621
 622 #ifdef HAVE_PTHREAD_H
 623         pthread_mutex_init(&pool->mutex, NULL);
 624 #endif
 625
 626         if (!prefix) {
 627                 modules = cf_item_parent(cf_sectiontoitem(parent));
 628                 if (modules) {
 629                         cs_name1 = cf_section_name1(modules);
 630                         if (cs_name1 && (strcmp(cs_name1, "modules") == 0)) {
 631                                 cs_name1 = cf_section_name1(parent);
 632                                 cs_name2 = cf_section_name2(parent);
 633                                 if (!cs_name2) {
 634                                         cs_name2 = cs_name1;
 635                                 }
 636
 637                                 lp_len = (sizeof(LOG_PREFIX) - 4) + strlen(cs_name1) + strlen(cs_name2);
 638                                 pool->log_prefix = rad_malloc(lp_len);
 639                                 snprintf(pool->log_prefix, lp_len, LOG_PREFIX, cs_name1,
 640                                          cs_name2);
 641                         }
 642                 } else {                /* not a module configuration */
 643                         cs_name1 = cf_section_name1(parent);
 644
 645                         pool->log_prefix = strdup(cs_name1);
 646                 }
 647         } else {
 648                 pool->log_prefix = strdup(prefix);
 649         }
 650
 651         DEBUG("%s: Initialising connection pool", pool->log_prefix);
 652
 653         if (cs) {
 654                 if (cf_section_parse(cs, pool, connection_config) < 0) {
 655                         goto error;
 656                 }
 657
 658                 if (cf_section_sub_find(cs, "trigger")) pool->trigger = true;
 659         } else {
 660                 pool->start = 5;
 661                 pool->min = 5;
 662                 pool->max = 10;
 663                 pool->spare = 3;
 664                 pool->cleanup_delay = 5;
 665                 pool->idle_timeout = 60;
 666         }
 667
 668         /*
 669          *      Some simple limits
 670          */
 671         if (pool->max > 1024) pool->max = 1024;
 672         if (pool->start > pool->max) pool->start = pool->max;
 673         if (pool->spare > (pool->max - pool->min)) {
 674                 pool->spare = pool->max - pool->min;
 675         }
 676         if ((pool->lifetime > 0) && (pool->idle_timeout > pool->lifetime)) {
 677                 pool->idle_timeout = 0;
 678         }
 679
 680         /*
 681          *      Create all of the connections, unless the admin says
 682          *      not to.
 683          */
 684         for (i = 0; i < pool->start; i++) {
 685                 this = fr_connection_spawn(pool, now);
 686                 if (!this) {
 687                 error:
 688                         fr_connection_pool_delete(pool);
 689                         return NULL;
 690                 }
 691         }
 692
 693         if (pool->trigger) exec_trigger(NULL, pool->cs, "start", true);
 694
 695         return pool;
 696 }
 697
 698
 699 /** Check whether a connection needs to be removed from the pool
 700  *
 701  * Will verify that the connection is within idle_timeout, max_uses, and
 702  * lifetime values. If it is not, the connection will be closed.
 703  *
 704  * @note Will only close connections not in use.
 705  * @note Must be called with the mutex held.
 706  *
 707  * @param[in,out] pool to modify.
 708  * @param[in,out] this Connection to manage.
 709  * @param[in] now Current time.
 710  * @return 0 if the connection was closed, otherwise 1.
 711  */
 712 static int fr_connection_manage(fr_connection_pool_t *pool,
 713                                 fr_connection_t *this,
 714                                 time_t now)
 715 {
 716         rad_assert(pool != NULL);
 717         rad_assert(this != NULL);
 718
 719         /*
 720          *      Don't terminated in-use connections
 721          */
 722         if (this->in_use) return 1;
 723
 724         if ((pool->max_uses > 0) &&
 725             (this->num_uses >= pool->max_uses)) {
 726                 DEBUG("%s: Closing expired connection (%" PRIu64 "): Hit max_uses limit", pool->log_prefix,
 727                       this->number);
 728         do_delete:
 729                 if ((pool->num <= pool->min) &&
 730                     (pool->last_complained < now)) {
 731                         WARN("%s: You probably need to lower \"min\"", pool->log_prefix);
 732
 733                         pool->last_complained = now;
 734                 }
 735                 fr_connection_close(pool, this);
 736                 return 0;
 737         }
 738
 739         if ((pool->lifetime > 0) &&
 740             ((this->created + pool->lifetime) < now)) {
 741                 DEBUG("%s: Closing expired connection (%" PRIu64 ")", pool->log_prefix, this->number);
 742                 goto do_delete;
 743         }
 744
 745         if ((pool->idle_timeout > 0) &&
 746             ((this->last_used + pool->idle_timeout) < now)) {
 747                 INFO("%s: Closing connection (%" PRIu64 "): Hit idle_timeout, was idle for %u seconds",
 748                      pool->log_prefix, this->number, (int) (now - this->last_used));
 749                 goto do_delete;
 750         }
 751
 752         return 1;
 753 }
 754
 755
 756 /** Check whether any connections needs to be removed from the pool
 757  *
 758  * Maintains the number of connections in the pool as per the configuration
 759  * parameters for the connection pool.
 760  *
 761  * @note Will only run checks the first time it's called in a given second,
 762  * to throttle connection spawning/closing.
 763  * @note Will only close connections not in use.
 764  * @note Must be called with the mutex held, will release mutex before
 765  * returning.
 766  *
 767  * @param[in,out] pool to manage.
 768  * @return 1
 769  */
 770 static int fr_connection_pool_check(fr_connection_pool_t *pool)
 771 {
 772         int spare, spawn;
 773         time_t now = time(NULL);
 774         fr_connection_t *this, *next;
 775
 776         if (pool->last_checked == now) {
 777                 pthread_mutex_unlock(&pool->mutex);
 778                 return 1;
 779         }
 780
 781         spare = pool->num - pool->active;
 782
 783         spawn = 0;
 784         if ((pool->num < pool->max) && (spare < pool->spare)) {
 785                 spawn = pool->spare - spare;
 786                 if ((spawn + pool->num) > pool->max) {
 787                         spawn = pool->max - pool->num;
 788                 }
 789                 if (pool->spawning) spawn = 0;
 790
 791                 if (spawn) {
 792                         pthread_mutex_unlock(&pool->mutex);
 793                         fr_connection_spawn(pool, now); /* ignore return code */
 794                         pthread_mutex_lock(&pool->mutex);
 795                 }
 796         }
 797
 798         /*
 799          *      We haven't spawned connections in a while, and there
 800          *      are too many spare ones.  Close the one which has been
 801          *      idle for the longest.
 802          */
 803         if ((now >= (pool->last_spawned + pool->cleanup_delay)) &&
 804             (spare > pool->spare)) {
 805                 fr_connection_t *idle;
 806
 807                 idle = NULL;
 808                 for (this = pool->tail; this != NULL; this = this->prev) {
 809                         if (this->in_use) continue;
 810
 811                         if (!idle ||
 812                            (this->last_used < idle->last_used)) {
 813                                 idle = this;
 814                         }
 815                 }
 816
 817                 rad_assert(idle != NULL);
 818
 819                 INFO("%s: Closing connection (%" PRIu64 "): Too many free connections (%d > %d)", pool->log_prefix,
 820                      idle->number, spare, pool->spare);
 821                 fr_connection_close(pool, idle);
 822         }
 823
 824         /*
 825          *      Pass over all of the connections in the pool, limiting
 826          *      lifetime, idle time, max requests, etc.
 827          */
 828         for (this = pool->head; this != NULL; this = next) {
 829                 next = this->next;
 830                 fr_connection_manage(pool, this, now);
 831         }
 832
 833         pool->last_checked = now;
 834         pthread_mutex_unlock(&pool->mutex);
 835
 836         return 1;
 837 }
 838
 839 /** Trigger connection check for a given connection or all connections
 840  *
 841  * If conn is not NULL then we call fr_connection_manage on the connection.
 842  * If conn is NULL we call fr_connection_pool_check on the pool.
 843  *
 844  * @note Only connections that are not in use will be closed.
 845  *
 846  * @see fr_connection_manage
 847  * @see fr_connection_pool_check
 848  * @param[in,out] pool to manage.
 849  * @param[in,out] conn to check.
 850  * @return 0 if the connection was closed, else 1.
 851  */
 852 int fr_connection_check(fr_connection_pool_t *pool, void *conn)
 853 {
 854         fr_connection_t *this;
 855         time_t now;
 856         int ret = 1;
 857
 858         if (!pool) return 1;
 859
 860         now = time(NULL);
 861         pthread_mutex_lock(&pool->mutex);
 862
 863         if (!conn) return fr_connection_pool_check(pool);
 864
 865         for (this = pool->head; this != NULL; this = this->next) {
 866                 if (this->connection == conn) {
 867                         ret = fr_connection_manage(pool, conn, now);
 868                         break;
 869                 }
 870         }
 871
 872         pthread_mutex_unlock(&pool->mutex);
 873
 874         return ret;
 875 }
 876
 877 /** Reserve a connection in the connection pool
 878  *
 879  * Will attempt to find an unused connection in the connection pool, if one is
 880  * found, will mark it as in in use increment the number of active connections
 881  * and return the connection handle.
 882  *
 883  * If no free connections are found will attempt to spawn a new one, conditional
 884  * on a connection spawning not already being in progress, and not being at the
 885  * 'max' connection limit.
 886  *
 887  * @note fr_connection_release must be called once the caller has finished
 888  * using the connection.
 889  *
 890  * @see fr_connection_release
 891  * @param[in,out] pool to reserve the connection from.
 892  * @return a pointer to the connection handle, or NULL on error.
 893  */
 894 void *fr_connection_get(fr_connection_pool_t *pool)
 895 {
 896         time_t now;
 897         fr_connection_t *this, *next;
 898
 899         if (!pool) return NULL;
 900
 901         pthread_mutex_lock(&pool->mutex);
 902
 903         now = time(NULL);
 904         for (this = pool->head; this != NULL; this = next) {
 905                 next = this->next;
 906
 907                 if (!this->in_use) goto do_return;
 908         }
 909
 910         if (pool->num == pool->max) {
 911                 int complain = false;
 912
 913                 /*
 914                  *      Rate-limit complaints.
 915                  */
 916                 if (pool->last_at_max != now) {
 917                         complain = true;
 918                         pool->last_at_max = now;
 919                 }
 920
 921                 pthread_mutex_unlock(&pool->mutex);
 922
 923                 if (complain) {
 924                         ERROR("%s: No connections available and at max connection limit", pool->log_prefix);
 925                 }
 926
 927                 return NULL;
 928         }
 929
 930         pthread_mutex_unlock(&pool->mutex);
 931         this = fr_connection_spawn(pool, now);
 932         if (!this) return NULL;
 933         pthread_mutex_lock(&pool->mutex);
 934
 935 do_return:
 936         pool->active++;
 937         this->num_uses++;
 938         this->last_used = now;
 939         this->in_use = true;
 940
 941         pthread_mutex_unlock(&pool->mutex);
 942
 943         DEBUG("%s: Reserved connection (%" PRIu64 ")", pool->log_prefix, this->number);
 944
 945         return this->connection;
 946 }
 947
 948 /** Release a connection
 949  *
 950  * Will mark a connection as unused and decrement the number of active
 951  * connections.
 952  *
 953  * @see fr_connection_get
 954  * @param[in,out] pool to release the connection in.
 955  * @param[in,out] conn to release.
 956  */
 957 void fr_connection_release(fr_connection_pool_t *pool, void *conn)
 958 {
 959         fr_connection_t *this;
 960
 961         this = fr_connection_find(pool, conn);
 962         if (!this) return;
 963
 964         rad_assert(this->in_use == true);
 965         this->in_use = false;
 966
 967         /*
 968          *      Determines whether the last used connection gets
 969          *      re-used first.
 970          */
 971         if (pool->spread) {
 972                 /*
 973                  *      Put it at the tail of the list, so
 974                  *      that it will get re-used last.
 975                  */
 976                 if (this != pool->tail) {
 977                         fr_connection_unlink(pool, this);
 978                         fr_connection_link_tail(pool, this);
 979                 }
 980         } else {
 981                 /*
 982                  *      Put it at the head of the list, so
 983                  *      that it will get re-used quickly.
 984                  */
 985                 if (this != pool->head) {
 986                         fr_connection_unlink(pool, this);
 987                         fr_connection_link_head(pool, this);
 988                 }
 989         }
 990
 991         rad_assert(pool->active > 0);
 992         pool->active--;
 993
 994         DEBUG("%s: Released connection (%" PRIu64 ")", pool->log_prefix, this->number);
 995
 996         /*
 997          *      We mirror the "spawn on get" functionality by having
 998          *      "delete on release".  If there are too many spare
 999          *      connections, go manage the pool && clean some up.
1000          */
1001         fr_connection_pool_check(pool);
1002 }
1003
1004 /** Reconnect a suspected inviable connection
1005  *
1006  * This should be called by the module if it suspects that a connection is
1007  * not viable (e.g. the server has closed it).
1008  *
1009  * Will attempt to create a new connection handle using the create callback,
1010  * and if this is successful the new handle will be assigned to the existing
1011  * pool connection.
1012  *
1013  * If this is not successful, the connection will be removed from the pool.
1014  *
1015  * When implementing a module that uses the connection pool API, it is advisable
1016  * to pass a pointer to the pointer to the handle (void **conn)
1017  * to all functions which may call reconnect. This is so that if a new handle
1018  * is created and returned, the handle pointer can be updated up the callstack,
1019  * and a function higher up the stack doesn't attempt to use a now invalid
1020  * connection handle.
1021  *
1022  * @warning After calling reconnect the caller *MUST NOT* attempt to use
1023  * the old handle in any other operations, as its memory will have been freed.
1024  *
1025  * @see fr_connection_get
1026  * @param[in,out] pool to reconnect the connection in.
1027  * @param[in,out] conn to reconnect.
1028  * @return ew connection handle if successful else NULL.
1029  */
1030 void *fr_connection_reconnect(fr_connection_pool_t *pool, void *conn)
1031 {
1032         void *new_conn;
1033         fr_connection_t *this;
1034         uint64_t conn_number;
1035
1036         if (!pool || !conn) return NULL;
1037
1038         this = fr_connection_find(pool, conn);
1039         if (!this) return NULL;
1040
1041         conn_number = this->number;
1042
1043         rad_assert(this->in_use == true);
1044
1045         DEBUG("%s: Reconnecting (%" PRIu64 ")", pool->log_prefix, conn_number);
1046
1047         new_conn = pool->create(pool->ctx);
1048         if (!new_conn) {
1049                 time_t now = time(NULL);
1050
1051                 if (pool->last_complained == now) {
1052                         now = 0;
1053                 } else {
1054                         pool->last_complained = now;
1055                 }
1056
1057                 this->in_use = false;
1058
1059                 rad_assert(pool->active > 0);
1060                 pool->active--;
1061
1062                 fr_connection_close(pool, this);
1063                 pthread_mutex_unlock(&pool->mutex);
1064
1065                 /*
1066                  *      Can't create a new socket.
1067                  *      Try grabbing a pre-existing one.
1068                  */
1069                 new_conn = fr_connection_get(pool);
1070                 if (new_conn) return new_conn;
1071
1072                 if (!now) return NULL;
1073
1074                 ERROR("%s: Failed to reconnect (%" PRIu64 "), and no other connections available", pool->log_prefix,
1075                       conn_number);
1076
1077                 return NULL;
1078         }
1079
1080         pool->delete(pool->ctx, conn);
1081         this->connection = new_conn;
1082         pthread_mutex_unlock(&pool->mutex);
1083         return new_conn;
1084 }