src/main/connection.c

   1 /*
   2  *   This program is free software; you can redistribute it and/or modify
   3  *   it under the terms of the GNU General Public License as published by
   4  *   the Free Software Foundation; either version 2 of the License, or
   5  *   (at your option) any later version.
   6  *
   7  *   This program is distributed in the hope that it will be useful,
   8  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  *   GNU General Public License for more details.
  11  *
  12  *   You should have received a copy of the GNU General Public License
  13  *   along with this program; if not, write to the Free Software
  14  *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  15  */
  16
  17 /**
  18  * @file connection.c
  19  * @brief Handle pools of connections (threads, sockets, etc.)
  20  * @note This API must be used by all modules in the public distribution that
  21  * maintain pools of connections.
  22  *
  23  * @copyright 2012  The FreeRADIUS server project
  24  * @copyright 2012  Alan DeKok <aland@deployingradius.com>
  25  */
  26 RCSID("$Id$")
  27
  28 #include <freeradius-devel/radiusd.h>
  29 #include <freeradius-devel/rad_assert.h>
  30
  31 typedef struct fr_connection fr_connection_t;
  32
  33 static int fr_connection_pool_check(fr_connection_pool_t *pool);
  34
  35 /** An individual connection within the connection pool
  36  *
  37  * Defines connection counters, timestamps, and holds a pointer to the
  38  * connection handle itself.
  39  *
  40  * @see fr_connection_pool_t
  41  */
  42 struct fr_connection {
  43         fr_connection_t *prev;          //!< Previous connection in list.
  44         fr_connection_t *next;          //!< Next connection in list.
  45
  46         time_t          created;        //!< Time connection was created.
  47         time_t          last_used;      //!< Last time the connection was
  48                                         //!< reserved.
  49
  50         uint64_t        num_uses;       //!< Number of times the connection
  51                                         //!< has been reserved.
  52         int             in_use;         //!< Whether the connection is currently
  53                                         //!< reserved.
  54         uint64_t        number;         //!< Unique ID assigned when the
  55                                         //!< connection is created, these will
  56                                         //!< monotonically increase over the
  57                                         //!< lifetime of the connection pool.
  58         void            *connection;    //!< Pointer to whatever the module
  59                                         //!< uses for a connection handle.
  60 };
  61
  62 /** A connection pool
  63  *
  64  * Defines the configuration of the connection pool, all the counters and
  65  * timestamps related to the connection pool, the mutex that stops multiple
  66  * threads leaving the pool in an inconsistent state, and the callbacks
  67  * required to open, close and check the status of connections within the pool.
  68  *
  69  * @see fr_connection
  70  */
  71 struct fr_connection_pool_t {
  72         int             start;          //!< Number of initial connections
  73         int             min;            //!< Minimum number of concurrent
  74                                         //!< connections to keep open.
  75         int             max;            //!< Maximum number of concurrent
  76                                         //!< connections to allow.
  77         int             spare;          //!< Number of spare connections to try
  78                                         //!< and maintain.
  79         int             cleanup_delay;  //!< Interval between pruning
  80                                         //!< excess connections.
  81         uint64_t        max_uses;       //!< Maximum number of times a
  82                                         //!< connection can be used before being
  83                                         //!< closed.
  84         int             lifetime;       //!< How long a connection can be open
  85                                         //!< before being closed (irrespective
  86                                         //!< of whether it's idle or not).
  87         int             idle_timeout;   //!< How long a connection can be idle
  88                                         //!< before being closed.
  89
  90         bool            trigger;        //!< If true execute connection triggers
  91                                         //!< associated with the connection
  92                                         //!< pool.
  93
  94         bool            spread;         //!< If true requests will be spread
  95                                         //!< across all connections, instead of
  96                                         //!< re-using the most recently used
  97                                         //!< connections first.
  98
  99         time_t          last_checked;   //!< Last time we pruned the connection
 100                                         //!< pool.
 101         time_t          last_spawned;   //!< Last time we spawned a connection.
 102         time_t          last_failed;    //!< Last time we tried to spawn a
 103                                         //!< a connection but failed.
 104         time_t          last_complained;//!< Last time we complained about
 105                                         //!< configuration parameters.
 106         time_t          last_throttled; //!< Last time we refused to spawn a
 107                                         //!< connection because the last
 108                                         //!< connection failed, or we were
 109                                         //!< already spawning a connection.
 110         time_t          last_at_max;    //!< Last time we hit the maximum number
 111                                         //!< of allowed connections.
 112
 113         uint64_t        count;          //!< Number of connections spawned over
 114                                         //!< the lifetime of the pool.
 115         int             num;            //!< Number of connections in the pool.
 116         int             active;         //!< Number of currently reserved
 117                                         //!< connections.
 118
 119         fr_connection_t *head;          //!< Start of the connection list.
 120         fr_connection_t *tail;          //!< End of the connection list.
 121
 122         bool            spawning;       //!< Whether we are currently attempting
 123                                         //!< to spawn a new connection.
 124
 125 #ifdef HAVE_PTHREAD_H
 126         pthread_mutex_t mutex;          //!< Mutex used to keep consistent state
 127                                         //!< when making modifications in
 128                                         //!< threaded mode.
 129 #endif
 130
 131         CONF_SECTION    *cs;            //!< Configuration section holding
 132                                         //!< the section of parsed config file
 133                                         //!< that relates to this pool.
 134         void            *ctx;           //!< Pointer to context data that will
 135                                         //!< be passed to callbacks.
 136
 137         char            *log_prefix;    //!< Log prefix to prepend to all log
 138                                         //!< messages created by the connection
 139                                         //!< pool code.
 140
 141         fr_connection_create_t  create; //!< Function used to create new
 142                                         //!< connections.
 143         fr_connection_alive_t   alive;  //!< Function used to check status
 144                                         //!< of connections.
 145         fr_connection_delete_t  delete; //!< Function used to close existing
 146                                         //!< connections.
 147 };
 148
 149 #define LOG_PREFIX "rlm_%s (%s)"
 150 #ifndef HAVE_PTHREAD_H
 151 #define pthread_mutex_lock(_x)
 152 #define pthread_mutex_unlock(_x)
 153 #endif
 154
 155 static const CONF_PARSER connection_config[] = {
 156         { "start",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, start),
 157           0, "5" },
 158         { "min",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, min),
 159           0, "5" },
 160         { "max",      PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max),
 161           0, "10" },
 162         { "spare",    PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, spare),
 163           0, "3" },
 164         { "uses",     PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, max_uses),
 165           0, "0" },
 166         { "lifetime", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, lifetime),
 167           0, "0" },
 168         { "cleanup_delay", PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, cleanup_delay),
 169           0, "5" },
 170         { "idle_timeout",  PW_TYPE_INTEGER, offsetof(fr_connection_pool_t, idle_timeout),
 171           0, "60" },
 172         { "spread", PW_TYPE_BOOLEAN, offsetof(fr_connection_pool_t, spread),
 173           0, "no" },
 174         { NULL, -1, 0, NULL, NULL }
 175 };
 176
 177 /** Removes a connection from the connection list
 178  *
 179  * @note Must be called with the mutex held.
 180  *
 181  * @param[in,out] pool to modify.
 182  * @param[in] this Connection to delete.
 183  */
 184 static void fr_connection_unlink(fr_connection_pool_t *pool,
 185                                  fr_connection_t *this)
 186 {
 187         if (this->prev) {
 188                 rad_assert(pool->head != this);
 189                 this->prev->next = this->next;
 190         } else {
 191                 rad_assert(pool->head == this);
 192                 pool->head = this->next;
 193         }
 194         if (this->next) {
 195                 rad_assert(pool->tail != this);
 196                 this->next->prev = this->prev;
 197         } else {
 198                 rad_assert(pool->tail == this);
 199                 pool->tail = this->prev;
 200         }
 201
 202         this->prev = this->next = NULL;
 203 }
 204
 205 /** Adds a connection to the head of the connection list
 206  *
 207  * @note Must be called with the mutex held.
 208  *
 209  * @param[in,out] pool to modify.
 210  * @param[in] this Connection to add.
 211  */
 212 static void fr_connection_link_head(fr_connection_pool_t *pool,
 213                                     fr_connection_t *this)
 214 {
 215         rad_assert(pool != NULL);
 216         rad_assert(this != NULL);
 217         rad_assert(pool->head != this);
 218         rad_assert(pool->tail != this);
 219
 220         if (pool->head) {
 221                 pool->head->prev = this;
 222         }
 223
 224         this->next = pool->head;
 225         this->prev = NULL;
 226         pool->head = this;
 227         if (!pool->tail) {
 228                 rad_assert(this->next == NULL);
 229                 pool->tail = this;
 230         } else {
 231                 rad_assert(this->next != NULL);
 232         }
 233 }
 234
 235 /** Adds a connection to the tail of the connection list
 236  *
 237  * @note Must be called with the mutex held.
 238  *
 239  * @param[in,out] pool to modify.
 240  * @param[in] this Connection to add.
 241  */
 242 static void fr_connection_link_tail(fr_connection_pool_t *pool,
 243                                     fr_connection_t *this)
 244 {
 245         rad_assert(pool != NULL);
 246         rad_assert(this != NULL);
 247         rad_assert(pool->head != this);
 248         rad_assert(pool->tail != this);
 249
 250         if (pool->tail) {
 251                 pool->tail->next = this;
 252         }
 253         this->prev = pool->tail;
 254         this->next = NULL;
 255         pool->tail = this;
 256         if (!pool->head) {
 257                 rad_assert(this->prev == NULL);
 258                 pool->head = this;
 259         } else {
 260                 rad_assert(this->prev != NULL);
 261         }
 262 }
 263
 264
 265 /** Spawns a new connection
 266  *
 267  * Spawns a new connection using the create callback, and returns it for
 268  * adding to the connection list.
 269  *
 270  * @note Will call the 'open' trigger.
 271  * @note Must be called with the mutex free.
 272  *
 273  * @param[in] pool to modify.
 274  * @param[in] now Current time.
 275  * @return the new connection struct or NULL on error.
 276  */
 277 static fr_connection_t *fr_connection_spawn(fr_connection_pool_t *pool,
 278                                             time_t now)
 279 {
 280         fr_connection_t *this;
 281         void *conn;
 282
 283         rad_assert(pool != NULL);
 284
 285         /*
 286          *      Prevent all threads from blocking if the resource
 287          *      were managing connections for appears to be unavailable.
 288          */
 289         if ((pool->num == 0) && pool->spawning) {
 290                 return NULL;
 291         }
 292
 293         pthread_mutex_lock(&pool->mutex);
 294         rad_assert(pool->num <= pool->max);
 295
 296         if ((pool->last_failed == now) || pool->spawning) {
 297                 bool complain = false;
 298
 299                 if (pool->last_throttled != now) {
 300                         complain = true;
 301
 302                         pool->last_throttled = now;
 303                 }
 304
 305                 pthread_mutex_unlock(&pool->mutex);
 306
 307                 if (complain) {
 308                         if (pool->spawning) {
 309                                 ERROR("%s: Cannot open new connection, "
 310                                        "connection spawning already in "
 311                                        "progress", pool->log_prefix);
 312                         } else {
 313                                 ERROR("%s: Last connection failed, "
 314                                        "throttling connection spawn",
 315                                        pool->log_prefix);
 316                         }
 317                 }
 318
 319                 return NULL;
 320         }
 321
 322         pool->spawning = true;
 323
 324         /*
 325          *      Unlock the mutex while we try to open a new
 326          *      connection.  If there are issues with the back-end,
 327          *      opening a new connection may take a LONG time.  In
 328          *      that case, we want the other connections to continue
 329          *      to be used.
 330          */
 331         pthread_mutex_unlock(&pool->mutex);
 332
 333         INFO("%s: Opening additional connection (%" PRIu64 ")", pool->log_prefix, pool->count);
 334
 335         this = rad_malloc(sizeof(*this));
 336         memset(this, 0, sizeof(*this));
 337
 338         /*
 339          *      This may take a long time, which prevents other
 340          *      threads from releasing connections.  We don't care
 341          *      about other threads opening new connections, as we
 342          *      already have no free connections.
 343          */
 344         conn = pool->create(pool->ctx);
 345         if (!conn) {
 346                 ERROR("%s: Opening connection failed (%" PRIu64 ")", pool->log_prefix, pool->count);
 347
 348                 pool->last_failed = now;
 349                 free(this);
 350                 pool->spawning = false; /* atomic, so no lock is needed */
 351                 return NULL;
 352         }
 353
 354         this->created = now;
 355         this->connection = conn;
 356
 357         /*
 358          *      And lock the mutex again while we link the new
 359          *      connection back into the pool.
 360          */
 361         pthread_mutex_lock(&pool->mutex);
 362
 363         this->number = pool->count++;
 364         this->last_used = now;
 365         fr_connection_link_head(pool, this);
 366         pool->num++;
 367         pool->spawning = false;
 368         pool->last_spawned = time(NULL);
 369
 370         pthread_mutex_unlock(&pool->mutex);
 371
 372         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 373
 374         return this;
 375 }
 376
 377 /** Add a new connection to the pool
 378  *
 379  * If conn is not NULL will attempt to add that connection handle to the pool.
 380  * If conn is NULL will attempt to spawn a new connection using the create
 381  * callback.
 382  *
 383  * @note Will call the 'open' trigger.
 384  *
 385  * @param[in,out] pool to add connection to.
 386  * @param[in] conn to add.
 387  * @return 0 if the connection wasn't added else 1.
 388  */
 389 int fr_connection_add(fr_connection_pool_t *pool, void *conn)
 390 {
 391         fr_connection_t *this;
 392
 393         if (!pool) return 0;
 394
 395         pthread_mutex_lock(&pool->mutex);
 396
 397         if (!conn) {
 398                 conn = pool->create(pool->ctx);
 399                 if (!conn) {
 400                         pthread_mutex_unlock(&pool->mutex);
 401                         return 0;
 402                 }
 403
 404                 INFO("%s: Opening connection successful (%" PRIu64 ")", pool->log_prefix, pool->count);
 405         }
 406
 407         /*
 408          *      Too many connections: can't add it.
 409          */
 410         if (pool->num >= pool->max) {
 411                 pthread_mutex_unlock(&pool->mutex);
 412                 return 0;
 413         }
 414
 415         this = rad_malloc(sizeof(*this));
 416         memset(this, 0, sizeof(*this));
 417
 418         this->created = time(NULL);
 419         this->connection = conn;
 420
 421         this->number = pool->count++;
 422         this->last_used = time(NULL);
 423         fr_connection_link_head(pool, this);
 424         pool->num++;
 425
 426         pthread_mutex_unlock(&pool->mutex);
 427
 428         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 429
 430         return 1;
 431 }
 432
 433 /** Close an existing connection.
 434  *
 435  * Removes the connection from the list, calls the delete callback to close
 436  * the connection, then frees memory allocated to the connection.
 437  *
 438  * @note Will call the 'close' trigger.
 439  * @note Must be called with the mutex held.
 440  *
 441  * @param[in,out] pool to modify.
 442  * @param[in,out] this Connection to delete.
 443
 444  */
 445 static void fr_connection_close(fr_connection_pool_t *pool,
 446                                 fr_connection_t *this)
 447 {
 448         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
 449
 450         rad_assert(this->in_use == false);
 451
 452         fr_connection_unlink(pool, this);
 453         pool->delete(pool->ctx, this->connection);
 454         rad_assert(pool->num > 0);
 455         pool->num--;
 456         free(this);
 457 }
 458
 459 /** Find a connection handle in the connection list
 460  *
 461  * Walks over the list of connections searching for a specified connection
 462  * handle and returns the first connection that contains that pointer.
 463  *
 464  * @note Will lock mutex and only release mutex if connection handle
 465  * is not found, so will usually return will mutex held.
 466  * @note Must be called with the mutex free.
 467  *
 468  * @param[in] pool to search in.
 469  * @param[in] conn handle to search for.
 470  * @return the connection containing the specified handle, or NULL if non is
 471  * found.
 472  */
 473 static fr_connection_t *fr_connection_find(fr_connection_pool_t *pool, void *conn)
 474 {
 475         fr_connection_t *this;
 476
 477         if (!pool || !conn) return NULL;
 478
 479         pthread_mutex_lock(&pool->mutex);
 480
 481         /*
 482          *      FIXME: This loop could be avoided if we passed a 'void
 483          *      **connection' instead.  We could use "offsetof" in
 484          *      order to find top of the parent structure.
 485          */
 486         for (this = pool->head; this != NULL; this = this->next) {
 487                 if (this->connection == conn) return this;
 488         }
 489
 490         pthread_mutex_unlock(&pool->mutex);
 491         return NULL;
 492 }
 493
 494 /** Delete a connection from the connection pool.
 495  *
 496  * Resolves the connection handle to a connection, then (if found)
 497  * closes, unlinks and frees that connection.
 498  *
 499  * @note Must be called with the mutex free.
 500  *
 501  * @param[in,out] pool Connection pool to modify.
 502  * @param[in] conn to delete.
 503  * @return 0 if the connection could not be found, else 1.
 504  */
 505 int fr_connection_del(fr_connection_pool_t *pool, void *conn)
 506 {
 507         fr_connection_t *this;
 508
 509         this = fr_connection_find(pool, conn);
 510         if (!this) return 0;
 511
 512         /*
 513          *      If it's in use, release it.
 514          */
 515         if (this->in_use) {
 516                 rad_assert(this->in_use == true);
 517                 this->in_use = false;
 518
 519                 rad_assert(pool->active > 0);
 520                 pool->active--;
 521         }
 522
 523         INFO("%s: Deleting connection (%" PRIu64 ")", pool->log_prefix, this->number);
 524
 525         fr_connection_close(pool, this);
 526         fr_connection_pool_check(pool);
 527         return 1;
 528 }
 529
 530 /** Delete a connection pool
 531  *
 532  * Closes, unlinks and frees all connections in the connection pool, then frees
 533  * all memory used by the connection pool.
 534  *
 535  * @note Will call the 'stop' trigger.
 536  * @note Must be called with the mutex free.
 537  *
 538  * @param[in,out] pool to delete.
 539  */
 540 void fr_connection_pool_delete(fr_connection_pool_t *pool)
 541 {
 542         fr_connection_t *this, *next;
 543
 544         if (!pool) return;
 545
 546         DEBUG("%s: Removing connection pool", pool->log_prefix);
 547
 548         pthread_mutex_lock(&pool->mutex);
 549
 550         for (this = pool->head; this != NULL; this = next) {
 551                 next = this->next;
 552
 553                 INFO("%s: Closing connection (%" PRIu64 ")", pool->log_prefix, this->number);
 554
 555                 fr_connection_close(pool, this);
 556         }
 557
 558         if (pool->trigger) exec_trigger(NULL, pool->cs, "stop", true);
 559
 560         rad_assert(pool->head == NULL);
 561         rad_assert(pool->tail == NULL);
 562         rad_assert(pool->num == 0);
 563
 564         free(pool->log_prefix);
 565         free(pool);
 566 }
 567
 568 /** Create a new connection pool
 569  *
 570  * Allocates structures used by the connection pool, initialises the various
 571  * configuration options and counters, and sets the callback functions.
 572  *
 573  * Will also spawn the number of connections specified by the 'start'
 574  * configuration options.
 575  *
 576  * @note Will call the 'start' trigger.
 577  *
 578  * @param[in] parent configuration section containing a 'pool' subsection.
 579  * @param[in] ctx pointer to pass to callbacks.
 580  * @param[in] c Callback to create new connections.
 581  * @param[in] a Callback to check the status of connections.
 582  * @param[in] d Callback to delete connections.
 583  * @param[in] prefix to prepend to all log message, if NULL will create prefix
 584  *      from parent conf section names.
 585  * @return A new connection pool or NULL on error.
 586  */
 587 fr_connection_pool_t *fr_connection_pool_init(CONF_SECTION *parent,
 588                                               void *ctx,
 589                                               fr_connection_create_t c,
 590                                               fr_connection_alive_t a,
 591                                               fr_connection_delete_t d,
 592                                               char *prefix)
 593 {
 594         int i, lp_len;
 595         fr_connection_pool_t *pool;
 596         fr_connection_t *this;
 597         CONF_SECTION *modules;
 598         CONF_SECTION *cs;
 599         char const *cs_name1, *cs_name2;
 600         time_t now = time(NULL);
 601
 602         if (!parent || !ctx || !c || !d) return NULL;
 603
 604         cs = cf_section_sub_find(parent, "pool");
 605         if (!cs) cs = cf_section_sub_find(parent, "limit");
 606
 607         pool = rad_malloc(sizeof(*pool));
 608         memset(pool, 0, sizeof(*pool));
 609
 610         pool->cs = cs;
 611         pool->ctx = ctx;
 612         pool->create = c;
 613         pool->alive = a;
 614         pool->delete = d;
 615
 616         pool->head = pool->tail = NULL;
 617
 618 #ifdef HAVE_PTHREAD_H
 619         pthread_mutex_init(&pool->mutex, NULL);
 620 #endif
 621
 622         if (!prefix) {
 623                 modules = cf_item_parent(cf_sectiontoitem(parent));
 624                 if (modules) {
 625                         cs_name1 = cf_section_name1(modules);
 626                         if (cs_name1 && (strcmp(cs_name1, "modules") == 0)) {
 627                                 cs_name1 = cf_section_name1(parent);
 628                                 cs_name2 = cf_section_name2(parent);
 629                                 if (!cs_name2) {
 630                                         cs_name2 = cs_name1;
 631                                 }
 632
 633                                 lp_len = (sizeof(LOG_PREFIX) - 4) + strlen(cs_name1) + strlen(cs_name2);
 634                                 pool->log_prefix = rad_malloc(lp_len);
 635                                 snprintf(pool->log_prefix, lp_len, LOG_PREFIX, cs_name1,
 636                                          cs_name2);
 637                         }
 638                 } else {                /* not a module configuration */
 639                         cs_name1 = cf_section_name1(parent);
 640
 641                         pool->log_prefix = strdup(cs_name1);
 642                 }
 643         } else {
 644                 pool->log_prefix = strdup(prefix);
 645         }
 646
 647         DEBUG("%s: Initialising connection pool", pool->log_prefix);
 648
 649         if (cs) {
 650                 if (cf_section_parse(cs, pool, connection_config) < 0) {
 651                         goto error;
 652                 }
 653
 654                 if (cf_section_sub_find(cs, "trigger")) pool->trigger = true;
 655         } else {
 656                 pool->start = 5;
 657                 pool->min = 5;
 658                 pool->max = 10;
 659                 pool->spare = 3;
 660                 pool->cleanup_delay = 5;
 661                 pool->idle_timeout = 60;
 662         }
 663
 664         /*
 665          *      Some simple limits
 666          */
 667         if (pool->max == 0) {
 668                 cf_log_err_cs(cs, "Cannot set 'max' to zero");
 669                 goto error;
 670         }
 671         if (pool->min > pool->max) {
 672                 cf_log_err_cs(cs, "Cannot set 'min' to more than 'max'");
 673                 goto error;
 674         }
 675
 676         if (pool->max > 1024) pool->max = 1024;
 677         if (pool->start > pool->max) pool->start = pool->max;
 678         if (pool->spare > (pool->max - pool->min)) {
 679                 pool->spare = pool->max - pool->min;
 680         }
 681         if ((pool->lifetime > 0) && (pool->idle_timeout > pool->lifetime)) {
 682                 pool->idle_timeout = 0;
 683         }
 684
 685         /*
 686          *      Create all of the connections, unless the admin says
 687          *      not to.
 688          */
 689         for (i = 0; i < pool->start; i++) {
 690                 this = fr_connection_spawn(pool, now);
 691                 if (!this) {
 692                 error:
 693                         fr_connection_pool_delete(pool);
 694                         return NULL;
 695                 }
 696         }
 697
 698         if (pool->trigger) exec_trigger(NULL, pool->cs, "start", true);
 699
 700         return pool;
 701 }
 702
 703
 704 /** Check whether a connection needs to be removed from the pool
 705  *
 706  * Will verify that the connection is within idle_timeout, max_uses, and
 707  * lifetime values. If it is not, the connection will be closed.
 708  *
 709  * @note Will only close connections not in use.
 710  * @note Must be called with the mutex held.
 711  *
 712  * @param[in,out] pool to modify.
 713  * @param[in,out] this Connection to manage.
 714  * @param[in] now Current time.
 715  * @return 0 if the connection was closed, otherwise 1.
 716  */
 717 static int fr_connection_manage(fr_connection_pool_t *pool,
 718                                 fr_connection_t *this,
 719                                 time_t now)
 720 {
 721         rad_assert(pool != NULL);
 722         rad_assert(this != NULL);
 723
 724         /*
 725          *      Don't terminated in-use connections
 726          */
 727         if (this->in_use) return 1;
 728
 729         if ((pool->max_uses > 0) &&
 730             (this->num_uses >= pool->max_uses)) {
 731                 DEBUG("%s: Closing expired connection (%" PRIu64 "): Hit max_uses limit", pool->log_prefix,
 732                       this->number);
 733         do_delete:
 734                 if ((pool->num <= pool->min) &&
 735                     (pool->last_complained < now)) {
 736                         WARN("%s: You probably need to lower \"min\"", pool->log_prefix);
 737
 738                         pool->last_complained = now;
 739                 }
 740                 fr_connection_close(pool, this);
 741                 return 0;
 742         }
 743
 744         if ((pool->lifetime > 0) &&
 745             ((this->created + pool->lifetime) < now)) {
 746                 DEBUG("%s: Closing expired connection (%" PRIu64 ")", pool->log_prefix, this->number);
 747                 goto do_delete;
 748         }
 749
 750         if ((pool->idle_timeout > 0) &&
 751             ((this->last_used + pool->idle_timeout) < now)) {
 752                 INFO("%s: Closing connection (%" PRIu64 "): Hit idle_timeout, was idle for %u seconds",
 753                      pool->log_prefix, this->number, (int) (now - this->last_used));
 754                 goto do_delete;
 755         }
 756
 757         return 1;
 758 }
 759
 760
 761 /** Check whether any connections needs to be removed from the pool
 762  *
 763  * Maintains the number of connections in the pool as per the configuration
 764  * parameters for the connection pool.
 765  *
 766  * @note Will only run checks the first time it's called in a given second,
 767  * to throttle connection spawning/closing.
 768  * @note Will only close connections not in use.
 769  * @note Must be called with the mutex held, will release mutex before
 770  * returning.
 771  *
 772  * @param[in,out] pool to manage.
 773  * @return 1
 774  */
 775 static int fr_connection_pool_check(fr_connection_pool_t *pool)
 776 {
 777         int spare, spawn;
 778         time_t now = time(NULL);
 779         fr_connection_t *this, *next;
 780
 781         if (pool->last_checked == now) {
 782                 pthread_mutex_unlock(&pool->mutex);
 783                 return 1;
 784         }
 785
 786         spare = pool->num - pool->active;
 787
 788         if ((pool->num < pool->max) && (spare < pool->spare)) {
 789                 spawn = pool->spare - spare;
 790                 if ((spawn + pool->num) > pool->max) {
 791                         spawn = pool->max - pool->num;
 792                 }
 793                 if (pool->spawning) spawn = 0;
 794
 795                 if (spawn) {
 796                         pthread_mutex_unlock(&pool->mutex);
 797                         fr_connection_spawn(pool, now); /* ignore return code */
 798                         pthread_mutex_lock(&pool->mutex);
 799                 }
 800         }
 801
 802         /*
 803          *      We haven't spawned connections in a while, and there
 804          *      are too many spare ones.  Close the one which has been
 805          *      idle for the longest.
 806          */
 807         if ((now >= (pool->last_spawned + pool->cleanup_delay)) &&
 808             (spare > pool->spare)) {
 809                 fr_connection_t *idle;
 810
 811                 idle = NULL;
 812                 for (this = pool->tail; this != NULL; this = this->prev) {
 813                         if (this->in_use) continue;
 814
 815                         if (!idle ||
 816                            (this->last_used < idle->last_used)) {
 817                                 idle = this;
 818                         }
 819                 }
 820
 821                 rad_assert(idle != NULL);
 822
 823                 INFO("%s: Closing connection (%" PRIu64 "): Too many free connections (%d > %d)", pool->log_prefix,
 824                      idle->number, spare, pool->spare);
 825                 fr_connection_close(pool, idle);
 826         }
 827
 828         /*
 829          *      Pass over all of the connections in the pool, limiting
 830          *      lifetime, idle time, max requests, etc.
 831          */
 832         for (this = pool->head; this != NULL; this = next) {
 833                 next = this->next;
 834                 fr_connection_manage(pool, this, now);
 835         }
 836
 837         pool->last_checked = now;
 838         pthread_mutex_unlock(&pool->mutex);
 839
 840         return 1;
 841 }
 842
 843 /** Trigger connection check for a given connection or all connections
 844  *
 845  * If conn is not NULL then we call fr_connection_manage on the connection.
 846  * If conn is NULL we call fr_connection_pool_check on the pool.
 847  *
 848  * @note Only connections that are not in use will be closed.
 849  *
 850  * @see fr_connection_manage
 851  * @see fr_connection_pool_check
 852  * @param[in,out] pool to manage.
 853  * @param[in,out] conn to check.
 854  * @return 0 if the connection was closed, else 1.
 855  */
 856 int fr_connection_check(fr_connection_pool_t *pool, void *conn)
 857 {
 858         fr_connection_t *this;
 859         time_t now;
 860         int ret = 1;
 861
 862         if (!pool) return 1;
 863
 864         now = time(NULL);
 865         pthread_mutex_lock(&pool->mutex);
 866
 867         if (!conn) return fr_connection_pool_check(pool);
 868
 869         for (this = pool->head; this != NULL; this = this->next) {
 870                 if (this->connection == conn) {
 871                         ret = fr_connection_manage(pool, conn, now);
 872                         break;
 873                 }
 874         }
 875
 876         pthread_mutex_unlock(&pool->mutex);
 877
 878         return ret;
 879 }
 880
 881 /** Reserve a connection in the connection pool
 882  *
 883  * Will attempt to find an unused connection in the connection pool, if one is
 884  * found, will mark it as in in use increment the number of active connections
 885  * and return the connection handle.
 886  *
 887  * If no free connections are found will attempt to spawn a new one, conditional
 888  * on a connection spawning not already being in progress, and not being at the
 889  * 'max' connection limit.
 890  *
 891  * @note fr_connection_release must be called once the caller has finished
 892  * using the connection.
 893  *
 894  * @see fr_connection_release
 895  * @param[in,out] pool to reserve the connection from.
 896  * @return a pointer to the connection handle, or NULL on error.
 897  */
 898 void *fr_connection_get(fr_connection_pool_t *pool)
 899 {
 900         time_t now;
 901         fr_connection_t *this, *next;
 902
 903         if (!pool) return NULL;
 904
 905         pthread_mutex_lock(&pool->mutex);
 906
 907         now = time(NULL);
 908         for (this = pool->head; this != NULL; this = next) {
 909                 next = this->next;
 910
 911                 if (!this->in_use) goto do_return;
 912         }
 913
 914         if (pool->num == pool->max) {
 915                 bool complain = false;
 916
 917                 /*
 918                  *      Rate-limit complaints.
 919                  */
 920                 if (pool->last_at_max != now) {
 921                         complain = true;
 922                         pool->last_at_max = now;
 923                 }
 924
 925                 pthread_mutex_unlock(&pool->mutex);
 926
 927                 if (complain) {
 928                         ERROR("%s: No connections available and at max connection limit", pool->log_prefix);
 929                 }
 930
 931                 return NULL;
 932         }
 933
 934         pthread_mutex_unlock(&pool->mutex);
 935         this = fr_connection_spawn(pool, now);
 936         if (!this) return NULL;
 937         pthread_mutex_lock(&pool->mutex);
 938
 939 do_return:
 940         pool->active++;
 941         this->num_uses++;
 942         this->last_used = now;
 943         this->in_use = true;
 944
 945         pthread_mutex_unlock(&pool->mutex);
 946
 947         DEBUG("%s: Reserved connection (%" PRIu64 ")", pool->log_prefix, this->number);
 948
 949         return this->connection;
 950 }
 951
 952 /** Release a connection
 953  *
 954  * Will mark a connection as unused and decrement the number of active
 955  * connections.
 956  *
 957  * @see fr_connection_get
 958  * @param[in,out] pool to release the connection in.
 959  * @param[in,out] conn to release.
 960  */
 961 void fr_connection_release(fr_connection_pool_t *pool, void *conn)
 962 {
 963         fr_connection_t *this;
 964
 965         this = fr_connection_find(pool, conn);
 966         if (!this) return;
 967
 968         rad_assert(this->in_use == true);
 969         this->in_use = false;
 970
 971         /*
 972          *      Determines whether the last used connection gets
 973          *      re-used first.
 974          */
 975         if (pool->spread) {
 976                 /*
 977                  *      Put it at the tail of the list, so
 978                  *      that it will get re-used last.
 979                  */
 980                 if (this != pool->tail) {
 981                         fr_connection_unlink(pool, this);
 982                         fr_connection_link_tail(pool, this);
 983                 }
 984         } else {
 985                 /*
 986                  *      Put it at the head of the list, so
 987                  *      that it will get re-used quickly.
 988                  */
 989                 if (this != pool->head) {
 990                         fr_connection_unlink(pool, this);
 991                         fr_connection_link_head(pool, this);
 992                 }
 993         }
 994
 995         rad_assert(pool->active > 0);
 996         pool->active--;
 997
 998         DEBUG("%s: Released connection (%" PRIu64 ")", pool->log_prefix, this->number);
 999
1000         /*
1001          *      We mirror the "spawn on get" functionality by having
1002          *      "delete on release".  If there are too many spare
1003          *      connections, go manage the pool && clean some up.
1004          */
1005         fr_connection_pool_check(pool);
1006 }
1007
1008 /** Reconnect a suspected inviable connection
1009  *
1010  * This should be called by the module if it suspects that a connection is
1011  * not viable (e.g. the server has closed it).
1012  *
1013  * Will attempt to create a new connection handle using the create callback,
1014  * and if this is successful the new handle will be assigned to the existing
1015  * pool connection.
1016  *
1017  * If this is not successful, the connection will be removed from the pool.
1018  *
1019  * When implementing a module that uses the connection pool API, it is advisable
1020  * to pass a pointer to the pointer to the handle (void **conn)
1021  * to all functions which may call reconnect. This is so that if a new handle
1022  * is created and returned, the handle pointer can be updated up the callstack,
1023  * and a function higher up the stack doesn't attempt to use a now invalid
1024  * connection handle.
1025  *
1026  * @warning After calling reconnect the caller *MUST NOT* attempt to use
1027  * the old handle in any other operations, as its memory will have been freed.
1028  *
1029  * @see fr_connection_get
1030  * @param[in,out] pool to reconnect the connection in.
1031  * @param[in,out] conn to reconnect.
1032  * @return ew connection handle if successful else NULL.
1033  */
1034 void *fr_connection_reconnect(fr_connection_pool_t *pool, void *conn)
1035 {
1036         void *new_conn;
1037         fr_connection_t *this;
1038         uint64_t conn_number;
1039
1040         if (!pool || !conn) return NULL;
1041
1042         this = fr_connection_find(pool, conn);
1043         if (!this) return NULL;
1044
1045         /*
1046          *      The pool is now locked.
1047          */
1048         conn_number = this->number;
1049
1050         DEBUG("%s: Reconnecting (%" PRIu64 ")", pool->log_prefix, conn_number);
1051
1052         new_conn = pool->create(pool->ctx);
1053         if (!new_conn) {
1054                 time_t now = time(NULL);
1055
1056                 if (pool->last_complained == now) {
1057                         now = 0;
1058                 } else {
1059                         pool->last_complained = now;
1060                 }
1061
1062                 /*
1063                  *      The caller has said the connection is bad.  So
1064                  *      if it's still in use, mark it as unused, and
1065                  *      close it.  If it's not in use, we just try to
1066                  *      get a new connection.
1067                  */
1068                 if (this->in_use) {
1069                         this->in_use = false;
1070
1071                         rad_assert(pool->active > 0);
1072                         pool->active--;
1073
1074                         fr_connection_close(pool, this);
1075                 }
1076
1077                 /*
1078                  *      We failed to create a new socket.
1079                  *      Try to grab an existing one.
1080                  */
1081                 pthread_mutex_unlock(&pool->mutex);
1082                 new_conn = fr_connection_get(pool);
1083                 if (new_conn) return new_conn;
1084
1085                 if (!now) return NULL;
1086
1087                 ERROR("%s: Failed to reconnect (%" PRIu64 "), and no other connections available", pool->log_prefix,
1088                       conn_number);
1089
1090                 return NULL;
1091         }
1092
1093         pool->delete(pool->ctx, conn);
1094         this->connection = new_conn;
1095         pthread_mutex_unlock(&pool->mutex);
1096         return new_conn;
1097 }