src/main/connection.c

   1 /*
   2  *   This program is free software; you can redistribute it and/or modify
   3  *   it under the terms of the GNU General Public License as published by
   4  *   the Free Software Foundation; either version 2 of the License, or
   5  *   (at your option) any later version.
   6  *
   7  *   This program is distributed in the hope that it will be useful,
   8  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  *   GNU General Public License for more details.
  11  *
  12  *   You should have received a copy of the GNU General Public License
  13  *   along with this program; if not, write to the Free Software
  14  *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  15  */
  16
  17 /**
  18  * @file connection.c
  19  * @brief Handle pools of connections (threads, sockets, etc.)
  20  * @note This API must be used by all modules in the public distribution that
  21  * maintain pools of connections.
  22  *
  23  * @copyright 2012  The FreeRADIUS server project
  24  * @copyright 2012  Alan DeKok <aland@deployingradius.com>
  25  */
  26 RCSID("$Id$")
  27
  28 #include <freeradius-devel/radiusd.h>
  29 #include <freeradius-devel/rad_assert.h>
  30
  31 typedef struct fr_connection fr_connection_t;
  32
  33 static int fr_connection_pool_check(fr_connection_pool_t *pool);
  34
  35 /** An individual connection within the connection pool
  36  *
  37  * Defines connection counters, timestamps, and holds a pointer to the
  38  * connection handle itself.
  39  *
  40  * @see fr_connection_pool_t
  41  */
  42 struct fr_connection {
  43         fr_connection_t *prev;          //!< Previous connection in list.
  44         fr_connection_t *next;          //!< Next connection in list.
  45
  46         time_t          created;        //!< Time connection was created.
  47         time_t          last_used;      //!< Last time the connection was
  48                                         //!< reserved.
  49
  50         uint64_t        num_uses;       //!< Number of times the connection
  51                                         //!< has been reserved.
  52         int             in_use;         //!< Whether the connection is currently
  53                                         //!< reserved.
  54         uint64_t        number;         //!< Unique ID assigned when the
  55                                         //!< connection is created, these will
  56                                         //!< monotonically increase over the
  57                                         //!< lifetime of the connection pool.
  58         void            *connection;    //!< Pointer to whatever the module
  59                                         //!< uses for a connection handle.
  60 };
  61
  62 /** A connection pool
  63  *
  64  * Defines the configuration of the connection pool, all the counters and
  65  * timestamps related to the connection pool, the mutex that stops multiple
  66  * threads leaving the pool in an inconsistent state, and the callbacks
  67  * required to open, close and check the status of connections within the pool.
  68  *
  69  * @see fr_connection
  70  */
  71 struct fr_connection_pool_t {
  72         int             start;          //!< Number of initial connections
  73         int             min;            //!< Minimum number of concurrent
  74                                         //!< connections to keep open.
  75         int             max;            //!< Maximum number of concurrent
  76                                         //!< connections to allow.
  77         int             spare;          //!< Number of spare connections to try
  78         int             cleanup_interval; //!< Initial timer for how
  79                                           //!< often we sweep the pool
  80                                           //!< for free connections.
  81                                           //!< (0 is infinite).
  82         int             delay_interval;  //!< When we next do a
  83                                         //!< cleanup.  Initialized to
  84                                         //!< cleanup_interval, and increase
  85                                         //!< from there based on the delay.
  86         int             next_delay;     //!< The next delay time.
  87                                         //!< cleanup.  Initialized to
  88                                         //!< cleanup_interval, and decays
  89                                         //!< from there.
  90         uint64_t        max_uses;       //!< Maximum number of times a
  91                                         //!< connection can be used before being
  92                                         //!< closed.
  93         int             lifetime;       //!< How long a connection can be open
  94                                         //!< before being closed (irrespective
  95                                         //!< of whether it's idle or not).
  96         int             idle_timeout;   //!< How long a connection can be idle
  97                                         //!< before being closed.
  98
  99         bool            trigger;        //!< If true execute connection triggers
 100                                         //!< associated with the connection
 101                                         //!< pool.
 102
 103         bool            spread;         //!< If true requests will be spread
 104                                         //!< across all connections, instead of
 105                                         //!< re-using the most recently used
 106                                         //!< connections first.
 107
 108         time_t          last_checked;   //!< Last time we pruned the connection
 109                                         //!< pool.
 110         time_t          last_spawned;   //!< Last time we spawned a connection.
 111         time_t          last_failed;    //!< Last time we tried to spawn a
 112                                         //!< a connection but failed.
 113         time_t          last_complained;//!< Last time we complained about
 114                                         //!< configuration parameters.
 115         time_t          last_throttled; //!< Last time we refused to spawn a
 116                                         //!< connection because the last
 117                                         //!< connection failed, or we were
 118                                         //!< already spawning a connection.
 119         time_t          last_at_max;    //!< Last time we hit the maximum number
 120                                         //!< of allowed connections.
 121
 122         uint64_t        count;          //!< Number of connections spawned over
 123                                         //!< the lifetime of the pool.
 124         int             num;            //!< Number of connections in the pool.
 125         int             active;         //!< Number of currently reserved
 126                                         //!< connections.
 127
 128         fr_connection_t *head;          //!< Start of the connection list.
 129         fr_connection_t *tail;          //!< End of the connection list.
 130
 131         bool            spawning;       //!< Whether we are currently attempting
 132                                         //!< to spawn a new connection.
 133
 134 #ifdef HAVE_PTHREAD_H
 135         pthread_mutex_t mutex;          //!< Mutex used to keep consistent state
 136                                         //!< when making modifications in
 137                                         //!< threaded mode.
 138 #endif
 139
 140         CONF_SECTION    *cs;            //!< Configuration section holding
 141                                         //!< the section of parsed config file
 142                                         //!< that relates to this pool.
 143         void            *ctx;           //!< Pointer to context data that will
 144                                         //!< be passed to callbacks.
 145
 146         char            *log_prefix;    //!< Log prefix to prepend to all log
 147                                         //!< messages created by the connection
 148                                         //!< pool code.
 149
 150         fr_connection_create_t  create; //!< Function used to create new
 151                                         //!< connections.
 152         fr_connection_alive_t   alive;  //!< Function used to check status
 153                                         //!< of connections.
 154         fr_connection_delete_t  delete; //!< Function used to close existing
 155                                         //!< connections.
 156 };
 157
 158 #define LOG_PREFIX "rlm_%s (%s)"
 159 #ifndef HAVE_PTHREAD_H
 160 #define pthread_mutex_lock(_x)
 161 #define pthread_mutex_unlock(_x)
 162 #endif
 163
 164 static const CONF_PARSER connection_config[] = {
 165         { "start",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, start),
 166           0, "5" },
 167         { "min",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, min),
 168           0, "5" },
 169         { "max",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max),
 170           0, "10" },
 171         { "spare",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, spare),
 172           0, "3" },
 173         { "uses",     PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max_uses),
 174           0, "0" },
 175         { "lifetime", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, lifetime),
 176           0, "0" },
 177         { "cleanup_delay", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, cleanup_interval),
 178           0, NULL},
 179         { "cleanup_interval", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, cleanup_interval),
 180           0, "30" },
 181         { "idle_timeout",  PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, idle_timeout),
 182           0, "60" },
 183         { "spread", PW_TYPE_BOOLEAN, offsetof(fr_connection_pool_t, spread),
 184           0, "no" },
 185         { NULL, -1, 0, NULL, NULL }
 186 };
 187
 188 /** Removes a connection from the connection list
 189  *
 190  * @note Must be called with the mutex held.
 191  *
 192  * @param[in,out] pool to modify.
 193  * @param[in] this Connection to delete.
 194  */
 195 static void fr_connection_unlink(fr_connection_pool_t *pool,
 196                                  fr_connection_t *this)
 197 {
 198         if (this->prev) {
 199                 rad_assert(pool->head != this);
 200                 this->prev->next = this->next;
 201         } else {
 202                 rad_assert(pool->head == this);
 203                 pool->head = this->next;
 204         }
 205         if (this->next) {
 206                 rad_assert(pool->tail != this);
 207                 this->next->prev = this->prev;
 208         } else {
 209                 rad_assert(pool->tail == this);
 210                 pool->tail = this->prev;
 211         }
 212
 213         this->prev = this->next = NULL;
 214 }
 215
 216 /** Adds a connection to the head of the connection list
 217  *
 218  * @note Must be called with the mutex held.
 219  *
 220  * @param[in,out] pool to modify.
 221  * @param[in] this Connection to add.
 222  */
 223 static void fr_connection_link_head(fr_connection_pool_t *pool,
 224                                     fr_connection_t *this)
 225 {
 226         rad_assert(pool != NULL);
 227         rad_assert(this != NULL);
 228         rad_assert(pool->head != this);
 229         rad_assert(pool->tail != this);
 230
 231         if (pool->head) {
 232                 pool->head->prev = this;
 233         }
 234
 235         this->next = pool->head;
 236         this->prev = NULL;
 237         pool->head = this;
 238         if (!pool->tail) {
 239                 rad_assert(this->next == NULL);
 240                 pool->tail = this;
 241         } else {
 242                 rad_assert(this->next != NULL);
 243         }
 244 }
 245
 246 /** Adds a connection to the tail of the connection list
 247  *
 248  * @note Must be called with the mutex held.
 249  *
 250  * @param[in,out] pool to modify.
 251  * @param[in] this Connection to add.
 252  */
 253 static void fr_connection_link_tail(fr_connection_pool_t *pool,
 254                                     fr_connection_t *this)
 255 {
 256         rad_assert(pool != NULL);
 257         rad_assert(this != NULL);
 258         rad_assert(pool->head != this);
 259         rad_assert(pool->tail != this);
 260
 261         if (pool->tail) {
 262                 pool->tail->next = this;
 263         }
 264         this->prev = pool->tail;
 265         this->next = NULL;
 266         pool->tail = this;
 267         if (!pool->head) {
 268                 rad_assert(this->prev == NULL);
 269                 pool->head = this;
 270         } else {
 271                 rad_assert(this->prev != NULL);
 272         }
 273 }
 274
 275
 276 /** Spawns a new connection
 277  *
 278  * Spawns a new connection using the create callback, and returns it for
 279  * adding to the connection list.
 280  *
 281  * @note Will call the 'open' trigger.
 282  * @note Must be called with the mutex free.
 283  *
 284  * @param[in] pool to modify.
 285  * @param[in] now Current time.
 286  * @return the new connection struct or NULL on error.
 287  */
 288 static fr_connection_t *fr_connection_spawn(fr_connection_pool_t *pool,
 289                                             time_t now)
 290 {
 291         fr_connection_t *this;
 292         void *conn;
 293
 294         rad_assert(pool != NULL);
 295
 296         /*
 297          *      Prevent all threads from blocking if the resource
 298          *      were managing connections for appears to be unavailable.
 299          */
 300         if ((pool->num == 0) && pool->spawning) {
 301                 return NULL;
 302         }
 303
 304         pthread_mutex_lock(&pool->mutex);
 305         rad_assert(pool->num <= pool->max);
 306
 307         if ((pool->last_failed == now) || pool->spawning) {
 308                 bool complain = false;
 309
 310                 if (pool->last_throttled != now) {
 311                         complain = true;
 312
 313                         pool->last_throttled = now;
 314                 }
 315
 316                 pthread_mutex_unlock(&pool->mutex);
 317
 318                 if (complain) {
 319                         if (pool->spawning) {
 320                                 ERROR("%s: Cannot open new connection, "
 321                                        "connection spawning already in "
 322                                        "progress", pool->log_prefix);
 323                         } else {
 324                                 ERROR("%s: Last connection failed, "
 325                                        "throttling connection spawn",
 326                                        pool->log_prefix);
 327                         }
 328                 }
 329
 330                 return NULL;
 331         }
 332
 333         pool->spawning = true;
 334
 335         /*
 336          *      Unlock the mutex while we try to open a new
 337          *      connection.  If there are issues with the back-end,
 338          *      opening a new connection may take a LONG time.  In
 339          *      that case, we want the other connections to continue
 340          *      to be used.
 341          */
 342         pthread_mutex_unlock(&pool->mutex);
 343
 344         INFO("%s: Opening additional connection (%" PRIu64 ")", pool->log_prefix, pool->count);
 345
 346         this = rad_malloc(sizeof(*this));
 347         memset(this, 0, sizeof(*this));
 348
 349         /*
 350          *      This may take a long time, which prevents other
 351          *      threads from releasing connections.  We don't care
 352          *      about other threads opening new connections, as we
 353          *      already have no free connections.
 354          */
 355         conn = pool->create(pool->ctx);
 356         if (!conn) {
 357                 ERROR("%s: Opening connection failed (%" PRIu64 ")", pool->log_prefix, pool->count);
 358
 359                 pool->last_failed = now;
 360                 free(this);
 361                 pool->spawning = false; /* atomic, so no lock is needed */
 362                 return NULL;
 363         }
 364
 365         this->created = now;
 366         this->connection = conn;
 367
 368         /*
 369          *      And lock the mutex again while we link the new
 370          *      connection back into the pool.
 371          */
 372         pthread_mutex_lock(&pool->mutex);
 373
 374         this->number = pool->count++;
 375         this->last_used = now;
 376         fr_connection_link_head(pool, this);
 377         pool->num++;
 378         pool->spawning = false;
 379         pool->last_spawned = time(NULL);
 380         pool->delay_interval = pool->cleanup_interval;
 381         pool->next_delay = pool->delay_interval + (pool->delay_interval >> 1);
 382
 383         pthread_mutex_unlock(&pool->mutex);
 384
 385         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 386
 387         return this;
 388 }
 389
 390 /** Add a new connection to the pool
 391  *
 392  * If conn is not NULL will attempt to add that connection handle to the pool.
 393  * If conn is NULL will attempt to spawn a new connection using the create
 394  * callback.
 395  *
 396  * @note Will call the 'open' trigger.
 397  *
 398  * @param[in,out] pool to add connection to.
 399  * @param[in] conn to add.
 400  * @return 0 if the connection wasn't added else 1.
 401  */
 402 int fr_connection_add(fr_connection_pool_t *pool, void *conn)
 403 {
 404         fr_connection_t *this;
 405
 406         if (!pool) return 0;
 407
 408         pthread_mutex_lock(&pool->mutex);
 409
 410         if (!conn) {
 411                 conn = pool->create(pool->ctx);
 412                 if (!conn) {
 413                         pthread_mutex_unlock(&pool->mutex);
 414                         return 0;
 415                 }
 416
 417                 INFO("%s: Opening connection successful (%" PRIu64 ")", pool->log_prefix, pool->count);
 418         }
 419
 420         /*
 421          *      Too many connections: can't add it.
 422          */
 423         if (pool->num >= pool->max) {
 424                 pthread_mutex_unlock(&pool->mutex);
 425                 return 0;
 426         }
 427
 428         this = rad_malloc(sizeof(*this));
 429         memset(this, 0, sizeof(*this));
 430
 431         this->created = time(NULL);
 432         this->connection = conn;
 433
 434         this->number = pool->count++;
 435         this->last_used = time(NULL);
 436         fr_connection_link_head(pool, this);
 437         pool->num++;
 438
 439         pthread_mutex_unlock(&pool->mutex);
 440
 441         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 442
 443         return 1;
 444 }
 445
 446 /** Close an existing connection.
 447  *
 448  * Removes the connection from the list, calls the delete callback to close
 449  * the connection, then frees memory allocated to the connection.
 450  *
 451  * @note Will call the 'close' trigger.
 452  * @note Must be called with the mutex held.
 453  *
 454  * @param[in,out] pool to modify.
 455  * @param[in,out] this Connection to delete.
 456
 457  */
 458 static void fr_connection_close(fr_connection_pool_t *pool,
 459                                 fr_connection_t *this)
 460 {
 461         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
 462
 463         rad_assert(this->in_use == false);
 464
 465         fr_connection_unlink(pool, this);
 466         pool->delete(pool->ctx, this->connection);
 467         rad_assert(pool->num > 0);
 468         pool->num--;
 469         free(this);
 470 }
 471
 472 /** Find a connection handle in the connection list
 473  *
 474  * Walks over the list of connections searching for a specified connection
 475  * handle and returns the first connection that contains that pointer.
 476  *
 477  * @note Will lock mutex and only release mutex if connection handle
 478  * is not found, so will usually return will mutex held.
 479  * @note Must be called with the mutex free.
 480  *
 481  * @param[in] pool to search in.
 482  * @param[in] conn handle to search for.
 483  * @return the connection containing the specified handle, or NULL if non is
 484  * found.
 485  */
 486 static fr_connection_t *fr_connection_find(fr_connection_pool_t *pool, void *conn)
 487 {
 488         fr_connection_t *this;
 489
 490         if (!pool || !conn) return NULL;
 491
 492         pthread_mutex_lock(&pool->mutex);
 493
 494         /*
 495          *      FIXME: This loop could be avoided if we passed a 'void
 496          *      **connection' instead.  We could use "offsetof" in
 497          *      order to find top of the parent structure.
 498          */
 499         for (this = pool->head; this != NULL; this = this->next) {
 500                 if (this->connection == conn) return this;
 501         }
 502
 503         pthread_mutex_unlock(&pool->mutex);
 504         return NULL;
 505 }
 506
 507 /** Delete a connection from the connection pool.
 508  *
 509  * Resolves the connection handle to a connection, then (if found)
 510  * closes, unlinks and frees that connection.
 511  *
 512  * @note Must be called with the mutex free.
 513  *
 514  * @param[in,out] pool Connection pool to modify.
 515  * @param[in] conn to delete.
 516  * @return 0 if the connection could not be found, else 1.
 517  */
 518 int fr_connection_del(fr_connection_pool_t *pool, void *conn)
 519 {
 520         fr_connection_t *this;
 521
 522         this = fr_connection_find(pool, conn);
 523         if (!this) return 0;
 524
 525         /*
 526          *      If it's in use, release it.
 527          */
 528         if (this->in_use) {
 529                 rad_assert(this->in_use == true);
 530                 this->in_use = false;
 531
 532                 rad_assert(pool->active > 0);
 533                 pool->active--;
 534         }
 535
 536         INFO("%s: Deleting connection (%" PRIu64 ")", pool->log_prefix, this->number);
 537
 538         fr_connection_close(pool, this);
 539         fr_connection_pool_check(pool);
 540         return 1;
 541 }
 542
 543 /** Delete a connection pool
 544  *
 545  * Closes, unlinks and frees all connections in the connection pool, then frees
 546  * all memory used by the connection pool.
 547  *
 548  * @note Will call the 'stop' trigger.
 549  * @note Must be called with the mutex free.
 550  *
 551  * @param[in,out] pool to delete.
 552  */
 553 void fr_connection_pool_delete(fr_connection_pool_t *pool)
 554 {
 555         fr_connection_t *this, *next;
 556
 557         if (!pool) return;
 558
 559         DEBUG("%s: Removing connection pool", pool->log_prefix);
 560
 561         pthread_mutex_lock(&pool->mutex);
 562
 563         for (this = pool->head; this != NULL; this = next) {
 564                 next = this->next;
 565
 566                 INFO("%s: Closing connection (%" PRIu64 ")", pool->log_prefix, this->number);
 567
 568                 fr_connection_close(pool, this);
 569         }
 570
 571         if (pool->trigger) exec_trigger(NULL, pool->cs, "stop", true);
 572
 573         rad_assert(pool->head == NULL);
 574         rad_assert(pool->tail == NULL);
 575         rad_assert(pool->num == 0);
 576
 577         free(pool->log_prefix);
 578         free(pool);
 579 }
 580
 581 /** Create a new connection pool
 582  *
 583  * Allocates structures used by the connection pool, initialises the various
 584  * configuration options and counters, and sets the callback functions.
 585  *
 586  * Will also spawn the number of connections specified by the 'start'
 587  * configuration options.
 588  *
 589  * @note Will call the 'start' trigger.
 590  *
 591  * @param[in] parent configuration section containing a 'pool' subsection.
 592  * @param[in] ctx pointer to pass to callbacks.
 593  * @param[in] c Callback to create new connections.
 594  * @param[in] a Callback to check the status of connections.
 595  * @param[in] d Callback to delete connections.
 596  * @param[in] prefix to prepend to all log message, if NULL will create prefix
 597  *      from parent conf section names.
 598  * @return A new connection pool or NULL on error.
 599  */
 600 fr_connection_pool_t *fr_connection_pool_init(CONF_SECTION *parent,
 601                                               void *ctx,
 602                                               fr_connection_create_t c,
 603                                               fr_connection_alive_t a,
 604                                               fr_connection_delete_t d,
 605                                               char *prefix)
 606 {
 607         int i, lp_len;
 608         fr_connection_pool_t *pool;
 609         fr_connection_t *this;
 610         CONF_SECTION *modules;
 611         CONF_SECTION *cs;
 612         char const *cs_name1, *cs_name2;
 613         time_t now = time(NULL);
 614
 615         if (!parent || !ctx || !c || !d) return NULL;
 616
 617         cs = cf_section_sub_find(parent, "pool");
 618         if (!cs) cs = cf_section_sub_find(parent, "limit");
 619
 620         pool = rad_malloc(sizeof(*pool));
 621         memset(pool, 0, sizeof(*pool));
 622
 623         pool->cs = cs;
 624         pool->ctx = ctx;
 625         pool->create = c;
 626         pool->alive = a;
 627         pool->delete = d;
 628
 629         pool->head = pool->tail = NULL;
 630
 631 #ifdef HAVE_PTHREAD_H
 632         pthread_mutex_init(&pool->mutex, NULL);
 633 #endif
 634
 635         if (!prefix) {
 636                 modules = cf_item_parent(cf_sectiontoitem(parent));
 637                 if (modules) {
 638                         cs_name1 = cf_section_name1(modules);
 639                         if (cs_name1 && (strcmp(cs_name1, "modules") == 0)) {
 640                                 cs_name1 = cf_section_name1(parent);
 641                                 cs_name2 = cf_section_name2(parent);
 642                                 if (!cs_name2) {
 643                                         cs_name2 = cs_name1;
 644                                 }
 645
 646                                 lp_len = (sizeof(LOG_PREFIX) - 4) + strlen(cs_name1) + strlen(cs_name2);
 647                                 pool->log_prefix = rad_malloc(lp_len);
 648                                 snprintf(pool->log_prefix, lp_len, LOG_PREFIX, cs_name1,
 649                                          cs_name2);
 650                         }
 651                 } else {                /* not a module configuration */
 652                         cs_name1 = cf_section_name1(parent);
 653
 654                         pool->log_prefix = strdup(cs_name1);
 655                 }
 656         } else {
 657                 pool->log_prefix = strdup(prefix);
 658         }
 659
 660         DEBUG("%s: Initialising connection pool", pool->log_prefix);
 661
 662         if (cs) {
 663                 if (cf_section_parse(cs, pool, connection_config) < 0) {
 664                         goto error;
 665                 }
 666
 667                 if (cf_section_sub_find(cs, "trigger")) pool->trigger = true;
 668         } else {
 669                 pool->start = 5;
 670                 pool->min = 5;
 671                 pool->max = 10;
 672                 pool->spare = 3;
 673                 pool->cleanup_interval = 30;
 674                 pool->idle_timeout = 60;
 675         }
 676
 677         /*
 678          *      Some simple limits
 679          */
 680         if (pool->max == 0) {
 681                 cf_log_err_cs(cs, "Cannot set 'max' to zero");
 682                 goto error;
 683         }
 684         if (pool->min > pool->max) {
 685                 cf_log_err_cs(cs, "Cannot set 'min' to more than 'max'");
 686                 goto error;
 687         }
 688
 689         if (pool->max > 1024) pool->max = 1024;
 690         if (pool->start > pool->max) pool->start = pool->max;
 691         if (pool->spare > (pool->max - pool->min)) {
 692                 pool->spare = pool->max - pool->min;
 693         }
 694         if ((pool->lifetime > 0) && (pool->idle_timeout > pool->lifetime)) {
 695                 pool->idle_timeout = 0;
 696         }
 697
 698         if (pool->cleanup_interval < 0) {
 699                 pool->cleanup_interval = 30;
 700         }
 701
 702         if ((pool->idle_timeout > 0) && (pool->cleanup_interval > pool->idle_timeout)) {
 703                 pool->cleanup_interval = pool->idle_timeout;
 704         }
 705
 706         /*
 707          *      Create all of the connections, unless the admin says
 708          *      not to.
 709          */
 710         for (i = 0; i < pool->start; i++) {
 711                 this = fr_connection_spawn(pool, now);
 712                 if (!this) {
 713                 error:
 714                         fr_connection_pool_delete(pool);
 715                         return NULL;
 716                 }
 717         }
 718
 719         if (pool->trigger) exec_trigger(NULL, pool->cs, "start", true);
 720
 721         return pool;
 722 }
 723
 724
 725 /** Check whether a connection needs to be removed from the pool
 726  *
 727  * Will verify that the connection is within idle_timeout, max_uses, and
 728  * lifetime values. If it is not, the connection will be closed.
 729  *
 730  * @note Will only close connections not in use.
 731  * @note Must be called with the mutex held.
 732  *
 733  * @param[in,out] pool to modify.
 734  * @param[in,out] this Connection to manage.
 735  * @param[in] now Current time.
 736  * @return 0 if the connection was closed, otherwise 1.
 737  */
 738 static int fr_connection_manage(fr_connection_pool_t *pool,
 739                                 fr_connection_t *this,
 740                                 time_t now)
 741 {
 742         rad_assert(pool != NULL);
 743         rad_assert(this != NULL);
 744
 745         /*
 746          *      Don't terminated in-use connections
 747          */
 748         if (this->in_use) return 1;
 749
 750         if ((pool->max_uses > 0) &&
 751             (this->num_uses >= pool->max_uses)) {
 752                 DEBUG("%s: Closing expired connection (%" PRIu64 "): Hit max_uses limit", pool->log_prefix,
 753                       this->number);
 754         do_delete:
 755                 if ((pool->num <= pool->min) &&
 756                     (pool->last_complained < now)) {
 757                         WARN("%s: You probably need to lower \"min\"", pool->log_prefix);
 758
 759                         pool->last_complained = now;
 760                 }
 761                 fr_connection_close(pool, this);
 762                 return 0;
 763         }
 764
 765         if ((pool->lifetime > 0) &&
 766             ((this->created + pool->lifetime) < now)) {
 767                 DEBUG("%s: Closing expired connection (%" PRIu64 ")", pool->log_prefix, this->number);
 768                 goto do_delete;
 769         }
 770
 771         if ((pool->idle_timeout > 0) &&
 772             ((this->last_used + pool->idle_timeout) < now)) {
 773                 INFO("%s: Closing connection (%" PRIu64 "): Hit idle_timeout, was idle for %u seconds",
 774                      pool->log_prefix, this->number, (int) (now - this->last_used));
 775                 goto do_delete;
 776         }
 777
 778         return 1;
 779 }
 780
 781
 782 /** Check whether any connections needs to be removed from the pool
 783  *
 784  * Maintains the number of connections in the pool as per the configuration
 785  * parameters for the connection pool.
 786  *
 787  * @note Will only run checks the first time it's called in a given second,
 788  * to throttle connection spawning/closing.
 789  * @note Will only close connections not in use.
 790  * @note Must be called with the mutex held, will release mutex before
 791  * returning.
 792  *
 793  * @param[in,out] pool to manage.
 794  * @return 1
 795  */
 796 static int fr_connection_pool_check(fr_connection_pool_t *pool)
 797 {
 798         int spare, spawn;
 799         time_t now = time(NULL);
 800         fr_connection_t *this, *next;
 801
 802         if (pool->last_checked == now) {
 803                 pthread_mutex_unlock(&pool->mutex);
 804                 return 1;
 805         }
 806
 807         spare = pool->num - pool->active;
 808
 809         if ((pool->num < pool->max) && (spare < pool->spare)) {
 810                 spawn = pool->spare - spare;
 811                 if ((spawn + pool->num) > pool->max) {
 812                         spawn = pool->max - pool->num;
 813                 }
 814                 if (pool->spawning) spawn = 0;
 815
 816                 if (spawn) {
 817                         pthread_mutex_unlock(&pool->mutex);
 818                         fr_connection_spawn(pool, now); /* ignore return code */
 819                         pthread_mutex_lock(&pool->mutex);
 820                 }
 821         }
 822
 823         /*
 824          *      We haven't spawned connections in a while, and there
 825          *      are too many spare ones.  Close the one which has been
 826          *      idle for the longest.
 827          */
 828         if ((now >= (pool->last_spawned + pool->delay_interval)) &&
 829             (spare > pool->spare)) {
 830                 fr_connection_t *idle;
 831
 832                 idle = NULL;
 833                 for (this = pool->tail; this != NULL; this = this->prev) {
 834                         if (this->in_use) continue;
 835
 836                         if (!idle ||
 837                            (this->last_used < idle->last_used)) {
 838                                 idle = this;
 839                         }
 840                 }
 841
 842                 rad_assert(idle != NULL);
 843
 844                 INFO("%s: Closing connection (%" PRIu64 "): Too many free connections (%d > %d)", pool->log_prefix,
 845                      idle->number, spare, pool->spare);
 846                 fr_connection_close(pool, idle);
 847
 848                 /*
 849                  *      Decrease the delay for the next time we clean
 850                  *      up.
 851                  */
 852                 pool->next_delay >>= 1;
 853                 if (pool->next_delay == 0) pool->next_delay = 1;
 854                 pool->delay_interval += pool->next_delay;
 855         }
 856
 857         /*
 858          *      Pass over all of the connections in the pool, limiting
 859          *      lifetime, idle time, max requests, etc.
 860          */
 861         for (this = pool->head; this != NULL; this = next) {
 862                 next = this->next;
 863                 fr_connection_manage(pool, this, now);
 864         }
 865
 866         pool->last_checked = now;
 867         pthread_mutex_unlock(&pool->mutex);
 868
 869         return 1;
 870 }
 871
 872 /** Trigger connection check for a given connection or all connections
 873  *
 874  * If conn is not NULL then we call fr_connection_manage on the connection.
 875  * If conn is NULL we call fr_connection_pool_check on the pool.
 876  *
 877  * @note Only connections that are not in use will be closed.
 878  *
 879  * @see fr_connection_manage
 880  * @see fr_connection_pool_check
 881  * @param[in,out] pool to manage.
 882  * @param[in,out] conn to check.
 883  * @return 0 if the connection was closed, else 1.
 884  */
 885 int fr_connection_check(fr_connection_pool_t *pool, void *conn)
 886 {
 887         fr_connection_t *this;
 888         time_t now;
 889         int ret = 1;
 890
 891         if (!pool) return 1;
 892
 893         now = time(NULL);
 894         pthread_mutex_lock(&pool->mutex);
 895
 896         if (!conn) return fr_connection_pool_check(pool);
 897
 898         for (this = pool->head; this != NULL; this = this->next) {
 899                 if (this->connection == conn) {
 900                         ret = fr_connection_manage(pool, conn, now);
 901                         break;
 902                 }
 903         }
 904
 905         pthread_mutex_unlock(&pool->mutex);
 906
 907         return ret;
 908 }
 909
 910 /** Reserve a connection in the connection pool
 911  *
 912  * Will attempt to find an unused connection in the connection pool, if one is
 913  * found, will mark it as in in use increment the number of active connections
 914  * and return the connection handle.
 915  *
 916  * If no free connections are found will attempt to spawn a new one, conditional
 917  * on a connection spawning not already being in progress, and not being at the
 918  * 'max' connection limit.
 919  *
 920  * @note fr_connection_release must be called once the caller has finished
 921  * using the connection.
 922  *
 923  * @see fr_connection_release
 924  * @param[in,out] pool to reserve the connection from.
 925  * @return a pointer to the connection handle, or NULL on error.
 926  */
 927 void *fr_connection_get(fr_connection_pool_t *pool)
 928 {
 929         time_t now;
 930         fr_connection_t *this, *next;
 931
 932         if (!pool) return NULL;
 933
 934         pthread_mutex_lock(&pool->mutex);
 935
 936         now = time(NULL);
 937         for (this = pool->head; this != NULL; this = next) {
 938                 next = this->next;
 939
 940                 if (!this->in_use) goto do_return;
 941         }
 942
 943         if (pool->num == pool->max) {
 944                 bool complain = false;
 945
 946                 /*
 947                  *      Rate-limit complaints.
 948                  */
 949                 if (pool->last_at_max != now) {
 950                         complain = true;
 951                         pool->last_at_max = now;
 952                 }
 953
 954                 pthread_mutex_unlock(&pool->mutex);
 955
 956                 if (complain) {
 957                         ERROR("%s: No connections available and at max connection limit", pool->log_prefix);
 958                 }
 959
 960                 return NULL;
 961         }
 962
 963         pthread_mutex_unlock(&pool->mutex);
 964         this = fr_connection_spawn(pool, now);
 965         if (!this) return NULL;
 966         pthread_mutex_lock(&pool->mutex);
 967
 968 do_return:
 969         pool->active++;
 970         this->num_uses++;
 971         this->last_used = now;
 972         this->in_use = true;
 973
 974         pthread_mutex_unlock(&pool->mutex);
 975
 976         DEBUG("%s: Reserved connection (%" PRIu64 ")", pool->log_prefix, this->number);
 977
 978         return this->connection;
 979 }
 980
 981 /** Release a connection
 982  *
 983  * Will mark a connection as unused and decrement the number of active
 984  * connections.
 985  *
 986  * @see fr_connection_get
 987  * @param[in,out] pool to release the connection in.
 988  * @param[in,out] conn to release.
 989  */
 990 void fr_connection_release(fr_connection_pool_t *pool, void *conn)
 991 {
 992         fr_connection_t *this;
 993
 994         this = fr_connection_find(pool, conn);
 995         if (!this) return;
 996
 997         rad_assert(this->in_use == true);
 998         this->in_use = false;
 999
1000         /*
1001          *      Determines whether the last used connection gets
1002          *      re-used first.
1003          */
1004         if (pool->spread) {
1005                 /*
1006                  *      Put it at the tail of the list, so
1007                  *      that it will get re-used last.
1008                  */
1009                 if (this != pool->tail) {
1010                         fr_connection_unlink(pool, this);
1011                         fr_connection_link_tail(pool, this);
1012                 }
1013         } else {
1014                 /*
1015                  *      Put it at the head of the list, so
1016                  *      that it will get re-used quickly.
1017                  */
1018                 if (this != pool->head) {
1019                         fr_connection_unlink(pool, this);
1020                         fr_connection_link_head(pool, this);
1021                 }
1022         }
1023
1024         rad_assert(pool->active > 0);
1025         pool->active--;
1026
1027         DEBUG("%s: Released connection (%" PRIu64 ")", pool->log_prefix, this->number);
1028
1029         /*
1030          *      We mirror the "spawn on get" functionality by having
1031          *      "delete on release".  If there are too many spare
1032          *      connections, go manage the pool && clean some up.
1033          */
1034         fr_connection_pool_check(pool);
1035 }
1036
1037 /** Reconnect a suspected inviable connection
1038  *
1039  * This should be called by the module if it suspects that a connection is
1040  * not viable (e.g. the server has closed it).
1041  *
1042  * Will attempt to create a new connection handle using the create callback,
1043  * and if this is successful the new handle will be assigned to the existing
1044  * pool connection.
1045  *
1046  * If this is not successful, the connection will be removed from the pool.
1047  *
1048  * When implementing a module that uses the connection pool API, it is advisable
1049  * to pass a pointer to the pointer to the handle (void **conn)
1050  * to all functions which may call reconnect. This is so that if a new handle
1051  * is created and returned, the handle pointer can be updated up the callstack,
1052  * and a function higher up the stack doesn't attempt to use a now invalid
1053  * connection handle.
1054  *
1055  * @warning After calling reconnect the caller *MUST NOT* attempt to use
1056  * the old handle in any other operations, as its memory will have been freed.
1057  *
1058  * @see fr_connection_get
1059  * @param[in,out] pool to reconnect the connection in.
1060  * @param[in,out] conn to reconnect.
1061  * @return ew connection handle if successful else NULL.
1062  */
1063 void *fr_connection_reconnect(fr_connection_pool_t *pool, void *conn)
1064 {
1065         void *new_conn;
1066         fr_connection_t *this;
1067         uint64_t conn_number;
1068
1069         if (!pool || !conn) return NULL;
1070
1071         this = fr_connection_find(pool, conn);
1072         if (!this) return NULL;
1073
1074         /*
1075          *      The pool is now locked.
1076          */
1077         conn_number = this->number;
1078
1079         DEBUG("%s: Reconnecting (%" PRIu64 ")", pool->log_prefix, conn_number);
1080
1081         new_conn = pool->create(pool->ctx);
1082         if (!new_conn) {
1083                 time_t now = time(NULL);
1084
1085                 if (pool->last_complained == now) {
1086                         now = 0;
1087                 } else {
1088                         pool->last_complained = now;
1089                 }
1090
1091                 /*
1092                  *      The caller has said the connection is bad.  So
1093                  *      if it's still in use, mark it as unused, and
1094                  *      close it.  If it's not in use, we just try to
1095                  *      get a new connection.
1096                  */
1097                 if (this->in_use) {
1098                         this->in_use = false;
1099
1100                         rad_assert(pool->active > 0);
1101                         pool->active--;
1102
1103                         fr_connection_close(pool, this);
1104                 }
1105
1106                 /*
1107                  *      We failed to create a new socket.
1108                  *      Try to grab an existing one.
1109                  */
1110                 pthread_mutex_unlock(&pool->mutex);
1111                 new_conn = fr_connection_get(pool);
1112                 if (new_conn) return new_conn;
1113
1114                 if (!now) return NULL;
1115
1116                 ERROR("%s: Failed to reconnect (%" PRIu64 "), and no other connections available", pool->log_prefix,
1117                       conn_number);
1118
1119                 return NULL;
1120         }
1121
1122         pool->delete(pool->ctx, conn);
1123         this->connection = new_conn;
1124         pthread_mutex_unlock(&pool->mutex);
1125         return new_conn;
1126 }