src/main/connection.c

   1 /*
   2  *   This program is free software; you can redistribute it and/or modify
   3  *   it under the terms of the GNU General Public License as published by
   4  *   the Free Software Foundation; either version 2 of the License, or
   5  *   (at your option) any later version.
   6  *
   7  *   This program is distributed in the hope that it will be useful,
   8  *   but WITHOUT ANY WARRANTY; without even the implied warranty of
   9  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  10  *   GNU General Public License for more details.
  11  *
  12  *   You should have received a copy of the GNU General Public License
  13  *   along with this program; if not, write to the Free Software
  14  *   Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  15  */
  16
  17 /**
  18  * @file connection.c
  19  * @brief Handle pools of connections (threads, sockets, etc.)
  20  * @note This API must be used by all modules in the public distribution that
  21  * maintain pools of connections.
  22  *
  23  * @copyright 2012  The FreeRADIUS server project
  24  * @copyright 2012  Alan DeKok <aland@deployingradius.com>
  25  */
  26 RCSID("$Id$")
  27
  28 #include <freeradius-devel/radiusd.h>
  29 #include <freeradius-devel/rad_assert.h>
  30
  31 typedef struct fr_connection fr_connection_t;
  32
  33 static int fr_connection_pool_check(fr_connection_pool_t *pool);
  34
  35 extern bool check_config;
  36
  37 #ifndef NDEBUG
  38 #ifdef HAVE_PTHREAD_H
  39 /* #define PTHREAD_DEBUG (1) */
  40 #endif
  41 #endif
  42
  43 /** An individual connection within the connection pool
  44  *
  45  * Defines connection counters, timestamps, and holds a pointer to the
  46  * connection handle itself.
  47  *
  48  * @see fr_connection_pool_t
  49  */
  50 struct fr_connection {
  51         fr_connection_t *prev;          //!< Previous connection in list.
  52         fr_connection_t *next;          //!< Next connection in list.
  53
  54         time_t          created;        //!< Time connection was created.
  55         time_t          last_used;      //!< Last time the connection was
  56                                         //!< reserved.
  57
  58         uint32_t        num_uses;       //!< Number of times the connection
  59                                         //!< has been reserved.
  60         uint64_t        number;         //!< Unique ID assigned when the
  61                                         //!< connection is created, these will
  62                                         //!< monotonically increase over the
  63                                         //!< lifetime of the connection pool.
  64         void            *connection;    //!< Pointer to whatever the module
  65                                         //!< uses for a connection handle.
  66         bool            in_use;         //!< Whether the connection is currently
  67                                         //!< reserved.
  68 #ifdef PTHREAD_DEBUG
  69         pthread_t       pthread_id;     //!< When 'in_use == true'
  70 #endif
  71 };
  72
  73 /** A connection pool
  74  *
  75  * Defines the configuration of the connection pool, all the counters and
  76  * timestamps related to the connection pool, the mutex that stops multiple
  77  * threads leaving the pool in an inconsistent state, and the callbacks
  78  * required to open, close and check the status of connections within the pool.
  79  *
  80  * @see fr_connection
  81  */
  82 struct fr_connection_pool_t {
  83         uint32_t        start;          //!< Number of initial connections
  84         uint32_t        min;            //!< Minimum number of concurrent
  85                                         //!< connections to keep open.
  86         uint32_t        max;            //!< Maximum number of concurrent
  87                                         //!< connections to allow.
  88         uint32_t        spare;          //!< Number of spare connections to try
  89         uint32_t        retry_delay;    //!< seconds to delay re-open
  90                                         //!< after a failed open.
  91         uint32_t        cleanup_interval; //!< Initial timer for how
  92                                           //!< often we sweep the pool
  93                                           //!< for free connections.
  94                                           //!< (0 is infinite).
  95         int             delay_interval;  //!< When we next do a
  96                                         //!< cleanup.  Initialized to
  97                                         //!< cleanup_interval, and increase
  98                                         //!< from there based on the delay.
  99         int             next_delay;     //!< The next delay time.
 100                                         //!< cleanup.  Initialized to
 101                                         //!< cleanup_interval, and decays
 102                                         //!< from there.
 103         uint64_t        max_uses;       //!< Maximum number of times a
 104                                         //!< connection can be used before being
 105                                         //!< closed.
 106         uint32_t        lifetime;       //!< How long a connection can be open
 107                                         //!< before being closed (irrespective
 108                                         //!< of whether it's idle or not).
 109         uint32_t        idle_timeout;   //!< How long a connection can be idle
 110                                         //!< before being closed.
 111
 112         bool            trigger;        //!< If true execute connection triggers
 113                                         //!< associated with the connection
 114                                         //!< pool.
 115
 116         bool            spread;         //!< If true requests will be spread
 117                                         //!< across all connections, instead of
 118                                         //!< re-using the most recently used
 119                                         //!< connections first.
 120
 121         time_t          last_checked;   //!< Last time we pruned the connection
 122                                         //!< pool.
 123         time_t          last_spawned;   //!< Last time we spawned a connection.
 124         time_t          last_failed;    //!< Last time we tried to spawn a
 125                                         //!< a connection but failed.
 126         time_t          last_throttled; //!< Last time we refused to spawn a
 127                                         //!< connection because the last
 128                                         //!< connection failed, or we were
 129                                         //!< already spawning a connection.
 130         time_t          last_at_max;    //!< Last time we hit the maximum number
 131                                         //!< of allowed connections.
 132
 133         uint64_t        count;          //!< Number of connections spawned over
 134                                         //!< the lifetime of the pool.
 135         uint32_t        num;            //!< Number of connections in the pool.
 136         int             active;         //!< Number of currently reserved
 137                                         //!< connections.
 138
 139         fr_connection_t *head;          //!< Start of the connection list.
 140         fr_connection_t *tail;          //!< End of the connection list.
 141
 142         bool            spawning;       //!< Whether we are currently attempting
 143                                         //!< to spawn a new connection.
 144
 145 #ifdef HAVE_PTHREAD_H
 146         pthread_mutex_t mutex;          //!< Mutex used to keep consistent state
 147                                         //!< when making modifications in
 148                                         //!< threaded mode.
 149 #endif
 150
 151         CONF_SECTION    *cs;            //!< Configuration section holding
 152                                         //!< the section of parsed config file
 153                                         //!< that relates to this pool.
 154         void            *opaque;        //!< Pointer to context data that will
 155                                         //!< be passed to callbacks.
 156
 157         char            *log_prefix;    //!< Log prefix to prepend to all log
 158                                         //!< messages created by the connection
 159                                         //!< pool code.
 160
 161         fr_connection_create_t  create; //!< Function used to create new
 162                                         //!< connections.
 163         fr_connection_alive_t   alive;  //!< Function used to check status
 164                                         //!< of connections.
 165         fr_connection_delete_t  delete; //!< Function used to close existing
 166                                         //!< connections.
 167 };
 168
 169 #define LOG_PREFIX "rlm_%s (%s)"
 170 #ifndef HAVE_PTHREAD_H
 171 #define pthread_mutex_lock(_x)
 172 #define pthread_mutex_unlock(_x)
 173 #endif
 174
 175 static const CONF_PARSER connection_config[] = {
 176         { "start", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, start), "5" },
 177         { "min", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, min), "5" },
 178         { "max", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, max), "10" },
 179         { "spare", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, spare), "3" },
 180         { "uses", FR_CONF_OFFSET(PW_TYPE_INTEGER64, fr_connection_pool_t, max_uses), "0" },
 181         { "lifetime", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, lifetime), "0" },
 182         { "cleanup_delay", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, cleanup_interval), NULL},
 183         { "cleanup_interval", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, cleanup_interval), "30" },
 184         { "idle_timeout", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, idle_timeout), "60" },
 185         { "retry_delay", FR_CONF_OFFSET(PW_TYPE_INTEGER, fr_connection_pool_t, retry_delay), "1" },
 186         { "spread", FR_CONF_OFFSET(PW_TYPE_BOOLEAN, fr_connection_pool_t, spread), "no" },
 187         { NULL, -1, 0, NULL, NULL }
 188 };
 189
 190 /** Removes a connection from the connection list
 191  *
 192  * @note Must be called with the mutex held.
 193  *
 194  * @param[in,out] pool to modify.
 195  * @param[in] this Connection to delete.
 196  */
 197 static void fr_connection_unlink(fr_connection_pool_t *pool,
 198                                  fr_connection_t *this)
 199 {
 200         if (this->prev) {
 201                 rad_assert(pool->head != this);
 202                 this->prev->next = this->next;
 203         } else {
 204                 rad_assert(pool->head == this);
 205                 pool->head = this->next;
 206         }
 207         if (this->next) {
 208                 rad_assert(pool->tail != this);
 209                 this->next->prev = this->prev;
 210         } else {
 211                 rad_assert(pool->tail == this);
 212                 pool->tail = this->prev;
 213         }
 214
 215         this->prev = this->next = NULL;
 216 }
 217
 218 /** Adds a connection to the head of the connection list
 219  *
 220  * @note Must be called with the mutex held.
 221  *
 222  * @param[in,out] pool to modify.
 223  * @param[in] this Connection to add.
 224  */
 225 static void fr_connection_link_head(fr_connection_pool_t *pool,
 226                                     fr_connection_t *this)
 227 {
 228         rad_assert(pool != NULL);
 229         rad_assert(this != NULL);
 230         rad_assert(pool->head != this);
 231         rad_assert(pool->tail != this);
 232
 233         if (pool->head) {
 234                 pool->head->prev = this;
 235         }
 236
 237         this->next = pool->head;
 238         this->prev = NULL;
 239         pool->head = this;
 240         if (!pool->tail) {
 241                 rad_assert(this->next == NULL);
 242                 pool->tail = this;
 243         } else {
 244                 rad_assert(this->next != NULL);
 245         }
 246 }
 247
 248 /** Adds a connection to the tail of the connection list
 249  *
 250  * @note Must be called with the mutex held.
 251  *
 252  * @param[in,out] pool to modify.
 253  * @param[in] this Connection to add.
 254  */
 255 static void fr_connection_link_tail(fr_connection_pool_t *pool,
 256                                     fr_connection_t *this)
 257 {
 258         rad_assert(pool != NULL);
 259         rad_assert(this != NULL);
 260         rad_assert(pool->head != this);
 261         rad_assert(pool->tail != this);
 262
 263         if (pool->tail) {
 264                 pool->tail->next = this;
 265         }
 266         this->prev = pool->tail;
 267         this->next = NULL;
 268         pool->tail = this;
 269         if (!pool->head) {
 270                 rad_assert(this->prev == NULL);
 271                 pool->head = this;
 272         } else {
 273                 rad_assert(this->prev != NULL);
 274         }
 275 }
 276
 277
 278 /** Spawns a new connection
 279  *
 280  * Spawns a new connection using the create callback, and returns it for
 281  * adding to the connection list.
 282  *
 283  * @note Will call the 'open' trigger.
 284  * @note Must be called with the mutex free.
 285  *
 286  * @param[in] pool to modify.
 287  * @param[in] now Current time.
 288  * @param[in] in_use whether the new connection should be "in_use" or not
 289  * @return the new connection struct or NULL on error.
 290  */
 291 static fr_connection_t *fr_connection_spawn(fr_connection_pool_t *pool,
 292                                             time_t now, bool in_use)
 293 {
 294         TALLOC_CTX *ctx;
 295
 296         fr_connection_t *this;
 297         void *conn;
 298
 299         rad_assert(pool != NULL);
 300
 301         /*
 302          *      Prevent all threads from blocking if the resource
 303          *      were managing connections for appears to be unavailable.
 304          */
 305         if ((pool->num == 0) && pool->spawning) {
 306                 return NULL;
 307         }
 308
 309         pthread_mutex_lock(&pool->mutex);
 310         rad_assert(pool->num <= pool->max);
 311
 312         /*
 313          *      Don't spawn multiple connections at the same time.
 314          */
 315         if (pool->spawning) {
 316                 pthread_mutex_unlock(&pool->mutex);
 317
 318                 ERROR("%s: Cannot open new connection, "
 319                       "connection spawning already in "
 320                       "progress", pool->log_prefix);
 321                 return NULL;
 322         }
 323
 324         /*
 325          *      If the last attempt failed, wait a bit before
 326          *      retrying.
 327          */
 328         if (pool->last_failed && ((pool->last_failed + pool->retry_delay) > now)) {
 329                 bool complain = false;
 330
 331                 if (pool->last_throttled != now) {
 332                         complain = true;
 333
 334                         pool->last_throttled = now;
 335                 }
 336
 337                 pthread_mutex_unlock(&pool->mutex);
 338
 339                 if (!RATE_LIMIT_ENABLED || complain) {
 340                         ERROR("%s: Last connection attempt failed, waiting %d seconds before retrying",
 341                               pool->log_prefix, pool->retry_delay);
 342                 }
 343
 344                 return NULL;
 345         }
 346
 347         pool->spawning = true;
 348
 349         /*
 350          *      Unlock the mutex while we try to open a new
 351          *      connection.  If there are issues with the back-end,
 352          *      opening a new connection may take a LONG time.  In
 353          *      that case, we want the other connections to continue
 354          *      to be used.
 355          */
 356         pthread_mutex_unlock(&pool->mutex);
 357
 358         INFO("%s: Opening additional connection (%" PRIu64 ")", pool->log_prefix, pool->count);
 359
 360         /*
 361          *      Allocate a new top level ctx for the create callback
 362          *      to hang its memory off of.
 363          */
 364         ctx = talloc_init("fr_connection_ctx");
 365         if (!ctx) return NULL;
 366
 367         /*
 368          *      This may take a long time, which prevents other
 369          *      threads from releasing connections.  We don't care
 370          *      about other threads opening new connections, as we
 371          *      already have no free connections.
 372          */
 373         conn = pool->create(ctx, pool->opaque);
 374         if (!conn) {
 375                 ERROR("%s: Opening connection failed (%" PRIu64 ")", pool->log_prefix, pool->count);
 376
 377                 pool->last_failed = now;
 378                 pool->spawning = false; /* atomic, so no lock is needed */
 379                 return NULL;
 380         }
 381
 382         /*
 383          *      And lock the mutex again while we link the new
 384          *      connection back into the pool.
 385          */
 386         pthread_mutex_lock(&pool->mutex);
 387
 388         this = talloc_zero(pool, fr_connection_t);
 389         if (!this) {
 390                 pthread_mutex_unlock(&pool->mutex);
 391                 return NULL;
 392         }
 393         fr_link_talloc_ctx_free(this, ctx);
 394
 395         this->created = now;
 396         this->connection = conn;
 397         this->in_use = in_use;
 398
 399         this->number = pool->count++;
 400         this->last_used = now;
 401         fr_connection_link_head(pool, this);
 402         pool->num++;
 403         pool->spawning = false;
 404         pool->last_spawned = time(NULL);
 405         pool->delay_interval = pool->cleanup_interval;
 406         pool->next_delay = pool->cleanup_interval;
 407         pool->last_failed = 0;
 408
 409         pthread_mutex_unlock(&pool->mutex);
 410
 411         if (pool->trigger) exec_trigger(NULL, pool->cs, "open", true);
 412
 413         return this;
 414 }
 415
 416 /** Close an existing connection.
 417  *
 418  * Removes the connection from the list, calls the delete callback to close
 419  * the connection, then frees memory allocated to the connection.
 420  *
 421  * @note Will call the 'close' trigger.
 422  * @note Must be called with the mutex held.
 423  *
 424  * @param[in,out] pool to modify.
 425  * @param[in,out] this Connection to delete.
 426
 427  */
 428 static void fr_connection_close(fr_connection_pool_t *pool,
 429                                 fr_connection_t *this)
 430 {
 431         /*
 432          *      If it's in use, release it.
 433          */
 434         if (this->in_use) {
 435 #ifdef PTHREAD_DEBUG
 436                 pthread_t pthread_id = pthread_self();
 437                 rad_assert(pthread_equal(this->pthread_id, pthread_id) != 0);
 438 #endif
 439                 this->in_use = false;
 440
 441                 rad_assert(pool->active > 0);
 442                 pool->active--;
 443         }
 444
 445         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
 446
 447         fr_connection_unlink(pool, this);
 448         if (pool->delete) pool->delete(pool->opaque, this->connection);
 449         rad_assert(pool->num > 0);
 450         pool->num--;
 451         talloc_free(this);
 452 }
 453
 454 /** Find a connection handle in the connection list
 455  *
 456  * Walks over the list of connections searching for a specified connection
 457  * handle and returns the first connection that contains that pointer.
 458  *
 459  * @note Will lock mutex and only release mutex if connection handle
 460  * is not found, so will usually return will mutex held.
 461  * @note Must be called with the mutex free.
 462  *
 463  * @param[in] pool to search in.
 464  * @param[in] conn handle to search for.
 465  * @return the connection containing the specified handle, or NULL if non is
 466  * found.
 467  */
 468 static fr_connection_t *fr_connection_find(fr_connection_pool_t *pool, void *conn)
 469 {
 470         fr_connection_t *this;
 471
 472         if (!pool || !conn) return NULL;
 473
 474         pthread_mutex_lock(&pool->mutex);
 475
 476         /*
 477          *      FIXME: This loop could be avoided if we passed a 'void
 478          *      **connection' instead.  We could use "offsetof" in
 479          *      order to find top of the parent structure.
 480          */
 481         for (this = pool->head; this != NULL; this = this->next) {
 482                 if (this->connection == conn) {
 483 #ifdef PTHREAD_DEBUG
 484                         pthread_t pthread_id;
 485
 486                         pthread_id = pthread_self();
 487                         rad_assert(pthread_equal(this->pthread_id, pthread_id) != 0);
 488 #endif
 489
 490                         rad_assert(this->in_use == true);
 491                         return this;
 492                 }
 493         }
 494
 495         pthread_mutex_unlock(&pool->mutex);
 496         return NULL;
 497 }
 498
 499 /** Delete a connection from the connection pool.
 500  *
 501  * Resolves the connection handle to a connection, then (if found)
 502  * closes, unlinks and frees that connection.
 503  *
 504  * @note Must be called with the mutex free.
 505  *
 506  * @param[in,out] pool Connection pool to modify.
 507  * @param[in] conn to delete.
 508  * @return 0 if the connection could not be found, else 1.
 509  */
 510 int fr_connection_del(fr_connection_pool_t *pool, void *conn)
 511 {
 512         fr_connection_t *this;
 513
 514         this = fr_connection_find(pool, conn);
 515         if (!this) return 0;
 516
 517         INFO("%s: Deleting connection (%" PRIu64 ")", pool->log_prefix, this->number);
 518
 519         fr_connection_close(pool, this);
 520         fr_connection_pool_check(pool);
 521         return 1;
 522 }
 523
 524 /** Delete a connection pool
 525  *
 526  * Closes, unlinks and frees all connections in the connection pool, then frees
 527  * all memory used by the connection pool.
 528  *
 529  * @note Will call the 'stop' trigger.
 530  * @note Must be called with the mutex free.
 531  *
 532  * @param[in,out] pool to delete.
 533  */
 534 void fr_connection_pool_delete(fr_connection_pool_t *pool)
 535 {
 536         fr_connection_t *this, *next;
 537
 538         if (!pool) return;
 539
 540         DEBUG("%s: Removing connection pool", pool->log_prefix);
 541
 542         pthread_mutex_lock(&pool->mutex);
 543
 544         for (this = pool->head; this != NULL; this = next) {
 545                 next = this->next;
 546
 547                 INFO("%s: Closing connection (%" PRIu64 ")", pool->log_prefix, this->number);
 548
 549                 fr_connection_close(pool, this);
 550         }
 551
 552         if (pool->trigger) exec_trigger(NULL, pool->cs, "stop", true);
 553
 554         rad_assert(pool->head == NULL);
 555         rad_assert(pool->tail == NULL);
 556         rad_assert(pool->num == 0);
 557
 558         talloc_free(pool);
 559 }
 560
 561 /** Create a new connection pool
 562  *
 563  * Allocates structures used by the connection pool, initialises the various
 564  * configuration options and counters, and sets the callback functions.
 565  *
 566  * Will also spawn the number of connections specified by the 'start'
 567  * configuration options.
 568  *
 569  * @note Will call the 'start' trigger.
 570  *
 571  * @param[in] parent configuration section containing a 'pool' subsection.
 572  * @param[in] opaque data pointer to pass to callbacks.
 573  * @param[in] c Callback to create new connections.
 574  * @param[in] a Callback to check the status of connections.
 575  * @param[in] d Callback to delete connections.
 576  * @param[in] prefix to prepend to all log message, if NULL will create prefix
 577  *      from parent conf section names.
 578  * @return A new connection pool or NULL on error.
 579  */
 580 fr_connection_pool_t *fr_connection_pool_init(CONF_SECTION *parent,
 581                                               void *opaque,
 582                                               fr_connection_create_t c,
 583                                               fr_connection_alive_t a,
 584                                               fr_connection_delete_t d,
 585                                               char const *prefix)
 586 {
 587         uint32_t i;
 588         fr_connection_pool_t *pool;
 589         fr_connection_t *this;
 590         CONF_SECTION *modules;
 591         CONF_SECTION *cs;
 592         char const *cs_name1, *cs_name2;
 593         time_t now = time(NULL);
 594
 595         if (!parent || !opaque || !c) return NULL;
 596
 597         cs = cf_section_sub_find(parent, "pool");
 598         if (!cs) cs = cf_section_sub_find(parent, "limit");
 599
 600         /*
 601          *      Pool is allocated in the NULL context as
 602          *      threads are likely to allocate memory
 603          *      beneath the pool.
 604          */
 605         pool = talloc_zero(NULL, fr_connection_pool_t);
 606         if (!pool) return NULL;
 607
 608         /*
 609          *      Ensure the pool is freed at the same time
 610          *      as its parent.
 611          */
 612         if (fr_link_talloc_ctx_free(cs ? cs : parent, pool) < 0) {
 613                 talloc_free(pool);
 614                 return NULL;
 615         }
 616
 617         pool->cs = cs;
 618         pool->opaque = opaque;
 619         pool->create = c;
 620         pool->alive = a;
 621         pool->delete = d;
 622
 623         pool->head = pool->tail = NULL;
 624
 625 #ifdef HAVE_PTHREAD_H
 626         pthread_mutex_init(&pool->mutex, NULL);
 627 #endif
 628
 629         if (!prefix) {
 630                 modules = cf_item_parent(cf_sectiontoitem(parent));
 631                 if (modules) {
 632                         cs_name1 = cf_section_name1(modules);
 633                         if (cs_name1 && (strcmp(cs_name1, "modules") == 0)) {
 634                                 cs_name1 = cf_section_name1(parent);
 635                                 cs_name2 = cf_section_name2(parent);
 636                                 if (!cs_name2) {
 637                                         cs_name2 = cs_name1;
 638                                 }
 639
 640                                 pool->log_prefix = talloc_typed_asprintf(pool, LOG_PREFIX, cs_name1,
 641                                                                    cs_name2);
 642                         }
 643                 } else {                /* not a module configuration */
 644                         cs_name1 = cf_section_name1(parent);
 645
 646                         pool->log_prefix = talloc_typed_strdup(pool, cs_name1);
 647                 }
 648         } else {
 649                 pool->log_prefix = talloc_typed_strdup(pool, prefix);
 650         }
 651
 652         DEBUG("%s: Initialising connection pool", pool->log_prefix);
 653
 654         if (cs) {
 655                 if (cf_section_parse(cs, pool, connection_config) < 0) {
 656                         goto error;
 657                 }
 658
 659                 if (cf_section_sub_find(cs, "trigger")) pool->trigger = true;
 660         } else {
 661                 pool->start = 5;
 662                 pool->min = 5;
 663                 pool->max = 10;
 664                 pool->spare = 3;
 665                 pool->cleanup_interval = 30;
 666                 pool->idle_timeout = 60;
 667         }
 668
 669         /*
 670          *      Some simple limits
 671          */
 672         if (pool->max == 0) {
 673                 cf_log_err_cs(cs, "Cannot set 'max' to zero");
 674                 goto error;
 675         }
 676         if (pool->min > pool->max) {
 677                 cf_log_err_cs(cs, "Cannot set 'min' to more than 'max'");
 678                 goto error;
 679         }
 680
 681         if (pool->max > 1024) pool->max = 1024;
 682         if (pool->start > pool->max) pool->start = pool->max;
 683         if (pool->spare > (pool->max - pool->min)) {
 684                 pool->spare = pool->max - pool->min;
 685         }
 686         if ((pool->lifetime > 0) && (pool->idle_timeout > pool->lifetime)) {
 687                 pool->idle_timeout = 0;
 688         }
 689
 690         if ((pool->idle_timeout > 0) && (pool->cleanup_interval > pool->idle_timeout)) {
 691                 pool->cleanup_interval = pool->idle_timeout;
 692         }
 693
 694         /*
 695          *      Don't open any connections.  Instead, force the limits
 696          *      to only 1 connection.
 697          *
 698          */
 699         if (check_config) {
 700                 pool->start = pool->min = pool->max = 1;
 701                 return pool;
 702         }
 703
 704         /*
 705          *      Create all of the connections, unless the admin says
 706          *      not to.
 707          */
 708         for (i = 0; i < pool->start; i++) {
 709                 this = fr_connection_spawn(pool, now, false);
 710                 if (!this) {
 711                 error:
 712                         fr_connection_pool_delete(pool);
 713                         return NULL;
 714                 }
 715         }
 716
 717         if (pool->trigger) exec_trigger(NULL, pool->cs, "start", true);
 718
 719         return pool;
 720 }
 721
 722
 723 /** Check whether a connection needs to be removed from the pool
 724  *
 725  * Will verify that the connection is within idle_timeout, max_uses, and
 726  * lifetime values. If it is not, the connection will be closed.
 727  *
 728  * @note Will only close connections not in use.
 729  * @note Must be called with the mutex held.
 730  *
 731  * @param[in,out] pool to modify.
 732  * @param[in,out] this Connection to manage.
 733  * @param[in] now Current time.
 734  * @return 0 if the connection was closed, otherwise 1.
 735  */
 736 static int fr_connection_manage(fr_connection_pool_t *pool,
 737                                 fr_connection_t *this,
 738                                 time_t now)
 739 {
 740         rad_assert(pool != NULL);
 741         rad_assert(this != NULL);
 742
 743         /*
 744          *      Don't terminated in-use connections
 745          */
 746         if (this->in_use) return 1;
 747
 748         if ((pool->max_uses > 0) &&
 749             (this->num_uses >= pool->max_uses)) {
 750                 DEBUG("%s: Closing expired connection (%" PRIu64 "): Hit max_uses limit", pool->log_prefix,
 751                       this->number);
 752         do_delete:
 753                 if (pool->num <= pool->min) {
 754                         RATE_LIMIT(WARN("%s: You probably need to lower \"min\"", pool->log_prefix));
 755                 }
 756                 fr_connection_close(pool, this);
 757                 return 0;
 758         }
 759
 760         if ((pool->lifetime > 0) &&
 761             ((this->created + pool->lifetime) < now)) {
 762                 DEBUG("%s: Closing expired connection (%" PRIu64 ")", pool->log_prefix, this->number);
 763                 goto do_delete;
 764         }
 765
 766         if ((pool->idle_timeout > 0) &&
 767             ((this->last_used + pool->idle_timeout) < now)) {
 768                 INFO("%s: Closing connection (%" PRIu64 "): Hit idle_timeout, was idle for %u seconds",
 769                      pool->log_prefix, this->number, (int) (now - this->last_used));
 770                 goto do_delete;
 771         }
 772
 773         return 1;
 774 }
 775
 776
 777 /** Check whether any connections needs to be removed from the pool
 778  *
 779  * Maintains the number of connections in the pool as per the configuration
 780  * parameters for the connection pool.
 781  *
 782  * @note Will only run checks the first time it's called in a given second,
 783  * to throttle connection spawning/closing.
 784  * @note Will only close connections not in use.
 785  * @note Must be called with the mutex held, will release mutex before
 786  * returning.
 787  *
 788  * @param[in,out] pool to manage.
 789  * @return 1
 790  */
 791 static int fr_connection_pool_check(fr_connection_pool_t *pool)
 792 {
 793         uint32_t spawn, idle, extra;
 794         time_t now = time(NULL);
 795         fr_connection_t *this, *next;
 796
 797         if (pool->last_checked == now) {
 798                 pthread_mutex_unlock(&pool->mutex);
 799                 return 1;
 800         }
 801
 802         /*
 803          *      Some idle connections are OK, if they're within the
 804          *      configured "spare" range.  Any extra connections
 805          *      outside of that range can be closed.
 806          */
 807         idle = pool->num - pool->active;
 808         if (idle <= pool->spare) {
 809                 extra = 0;
 810         } else {
 811                 extra = idle - pool->spare;
 812         }
 813
 814         /*
 815          *      The other end can close connections.  If so, we'll
 816          *      have fewer than "min".  When that happens, open more
 817          *      connections to enforce "min".
 818          */
 819         if (pool->num <= pool->min) {
 820                 if (pool->spawning) {
 821                         spawn = 0;
 822                 } else {
 823                         spawn = pool->min - pool->num;
 824                 }
 825                 extra = 0;
 826
 827         } else if (pool->num >= pool->max) {
 828                 /*
 829                  *      Ensure we don't spawn more connections.  If
 830                  *      there are extra idle connections, we can
 831                  *      delete all of them.
 832                  */
 833                 spawn = 0;
 834                 /* leave extra alone from above */
 835
 836         } else if (idle <= pool->spare) {
 837                 /*
 838                  *      Not enough spare connections.  Spawn a few.
 839                  *      But cap the pool size at "max"
 840                  */
 841                 spawn = pool->spare - idle;
 842                 extra = 0;
 843
 844                 if ((pool->num + spawn) > pool->max) {
 845                         spawn = pool->max - pool->num;
 846                 }
 847
 848         } else if ((pool->min + extra) >= pool->num) {
 849                 /*
 850                  *      If closing the extra connections would take us
 851                  *      below "min", then don't do that.  Cap the
 852                  *      spare connections at the ones which will take
 853                  *      us exactly to "min".
 854                  */
 855                 spawn = 0;
 856                 extra = pool->num - pool->min;
 857
 858         } else {
 859                 /*
 860                  *      Closing the "extra" connections won't take us
 861                  *      below "min".  It's therefore safe to close
 862                  *      them all.
 863                  */
 864                 spawn = 0;
 865                 /* leave extra alone from above */
 866         }
 867
 868         if (spawn) {
 869                 pthread_mutex_unlock(&pool->mutex);
 870                 fr_connection_spawn(pool, now, false); /* ignore return code */
 871                 pthread_mutex_lock(&pool->mutex);
 872         }
 873
 874         /*
 875          *      We haven't spawned connections in a while, and there
 876          *      are too many spare ones.  Close the one which has been
 877          *      unused for the longest.
 878          */
 879         if (extra && (now >= (pool->last_spawned + pool->delay_interval))) {
 880                 fr_connection_t *found;
 881
 882                 found = NULL;
 883                 for (this = pool->tail; this != NULL; this = this->prev) {
 884                         if (this->in_use) continue;
 885
 886                         if (!found ||
 887                            (this->last_used < found->last_used)) {
 888                                 found = this;
 889                         }
 890                 }
 891
 892                 rad_assert(found != NULL);
 893
 894                 INFO("%s: Closing connection (%" PRIu64 "), from %d unused connections", pool->log_prefix,
 895                      found->number, extra);
 896                 fr_connection_close(pool, found);
 897
 898                 /*
 899                  *      Decrease the delay for the next time we clean
 900                  *      up.
 901                  */
 902                 pool->next_delay >>= 1;
 903                 if (pool->next_delay == 0) pool->next_delay = 1;
 904                 pool->delay_interval += pool->next_delay;
 905         }
 906
 907         /*
 908          *      Pass over all of the connections in the pool, limiting
 909          *      lifetime, idle time, max requests, etc.
 910          */
 911         for (this = pool->head; this != NULL; this = next) {
 912                 next = this->next;
 913                 fr_connection_manage(pool, this, now);
 914         }
 915
 916         pool->last_checked = now;
 917         pthread_mutex_unlock(&pool->mutex);
 918
 919         return 1;
 920 }
 921
 922 /** Get a connection from the connection pool
 923  *
 924  * @param[in,out] pool to reserve the connection from.
 925  * @param[in] spawn whether to spawn a new connection
 926  * @return a pointer to the connection handle, or NULL on error.
 927  */
 928 static void *fr_connection_get_internal(fr_connection_pool_t *pool, int spawn)
 929 {
 930         time_t now;
 931         fr_connection_t *this, *next;
 932
 933         if (!pool) return NULL;
 934
 935         pthread_mutex_lock(&pool->mutex);
 936
 937         now = time(NULL);
 938         for (this = pool->head; this != NULL; this = next) {
 939                 next = this->next;
 940
 941                 if (!this->in_use) goto do_return;
 942         }
 943
 944         if (pool->num == pool->max) {
 945                 bool complain = false;
 946
 947                 /*
 948                  *      Rate-limit complaints.
 949                  */
 950                 if (pool->last_at_max != now) {
 951                         complain = true;
 952                         pool->last_at_max = now;
 953                 }
 954
 955                 pthread_mutex_unlock(&pool->mutex);
 956
 957                 if (!RATE_LIMIT_ENABLED || complain) {
 958                         ERROR("%s: No connections available and at max connection limit", pool->log_prefix);
 959                 }
 960
 961                 return NULL;
 962         }
 963
 964         pthread_mutex_unlock(&pool->mutex);
 965
 966         if (!spawn) return NULL;
 967
 968         this = fr_connection_spawn(pool, now, true); /* MY connection! */
 969         if (!this) return NULL;
 970         pthread_mutex_lock(&pool->mutex);
 971
 972 do_return:
 973         pool->active++;
 974         this->num_uses++;
 975         this->last_used = now;
 976         this->in_use = true;
 977
 978 #ifdef PTHREAD_DEBUG
 979         this->pthread_id = pthread_self();
 980 #endif
 981         pthread_mutex_unlock(&pool->mutex);
 982
 983         DEBUG("%s: Reserved connection (%" PRIu64 ")", pool->log_prefix, this->number);
 984
 985         return this->connection;
 986 }
 987
 988
 989 /** Reserve a connection in the connection pool
 990  *
 991  * Will attempt to find an unused connection in the connection pool, if one is
 992  * found, will mark it as in in use increment the number of active connections
 993  * and return the connection handle.
 994  *
 995  * If no free connections are found will attempt to spawn a new one, conditional
 996  * on a connection spawning not already being in progress, and not being at the
 997  * 'max' connection limit.
 998  *
 999  * @note fr_connection_release must be called once the caller has finished
1000  * using the connection.
1001  *
1002  * @see fr_connection_release
1003  * @param[in,out] pool to reserve the connection from.
1004  * @return a pointer to the connection handle, or NULL on error.
1005  */
1006 void *fr_connection_get(fr_connection_pool_t *pool)
1007 {
1008         return fr_connection_get_internal(pool, true);
1009 }
1010
1011 /** Get the number of connections currently in the pool
1012  *
1013  * @param pool to count connections for.
1014  * @return the number of connections in the pool
1015  */
1016 int fr_connection_get_num(fr_connection_pool_t *pool)
1017 {
1018         return pool->num;
1019 }
1020
1021 /** Release a connection
1022  *
1023  * Will mark a connection as unused and decrement the number of active
1024  * connections.
1025  *
1026  * @see fr_connection_get
1027  * @param[in,out] pool to release the connection in.
1028  * @param[in,out] conn to release.
1029  */
1030 void fr_connection_release(fr_connection_pool_t *pool, void *conn)
1031 {
1032         fr_connection_t *this;
1033
1034         this = fr_connection_find(pool, conn);
1035         if (!this) return;
1036
1037         this->in_use = false;
1038
1039         /*
1040          *      Determines whether the last used connection gets
1041          *      re-used first.
1042          */
1043         if (pool->spread) {
1044                 /*
1045                  *      Put it at the tail of the list, so
1046                  *      that it will get re-used last.
1047                  */
1048                 if (this != pool->tail) {
1049                         fr_connection_unlink(pool, this);
1050                         fr_connection_link_tail(pool, this);
1051                 }
1052         } else {
1053                 /*
1054                  *      Put it at the head of the list, so
1055                  *      that it will get re-used quickly.
1056                  */
1057                 if (this != pool->head) {
1058                         fr_connection_unlink(pool, this);
1059                         fr_connection_link_head(pool, this);
1060                 }
1061         }
1062
1063         rad_assert(pool->active > 0);
1064         pool->active--;
1065
1066         DEBUG("%s: Released connection (%" PRIu64 ")", pool->log_prefix, this->number);
1067
1068         /*
1069          *      We mirror the "spawn on get" functionality by having
1070          *      "delete on release".  If there are too many spare
1071          *      connections, go manage the pool && clean some up.
1072          */
1073         fr_connection_pool_check(pool);
1074 }
1075
1076 /** Reconnect a suspected inviable connection
1077  *
1078  * This should be called by the module if it suspects that a connection is
1079  * not viable (e.g. the server has closed it).
1080  *
1081  * Will attempt to create a new connection handle using the create callback,
1082  * and if this is successful the new handle will be assigned to the existing
1083  * pool connection.
1084  *
1085  * If this is not successful, the connection will be removed from the pool.
1086  *
1087  * When implementing a module that uses the connection pool API, it is advisable
1088  * to pass a pointer to the pointer to the handle (void **conn)
1089  * to all functions which may call reconnect. This is so that if a new handle
1090  * is created and returned, the handle pointer can be updated up the callstack,
1091  * and a function higher up the stack doesn't attempt to use a now invalid
1092  * connection handle.
1093  *
1094  * @warning After calling reconnect the caller *MUST NOT* attempt to use
1095  * the old handle in any other operations, as its memory will have been freed.
1096  *
1097  * @see fr_connection_get
1098  * @param[in,out] pool to reconnect the connection in.
1099  * @param[in,out] conn to reconnect.
1100  * @return new connection handle if successful else NULL.
1101  */
1102 void *fr_connection_reconnect(fr_connection_pool_t *pool, void *conn)
1103 {
1104         void *new_conn;
1105         fr_connection_t *this;
1106         uint64_t conn_number;
1107         TALLOC_CTX *ctx;
1108
1109         if (!pool || !conn) return NULL;
1110
1111         /*
1112          *      If fr_connection_find is successful the pool is now locked
1113          */
1114         this = fr_connection_find(pool, conn);
1115         if (!this) return NULL;
1116
1117
1118         conn_number = this->number;
1119
1120         /*
1121          *      Destroy any handles associated with the fr_connection_t
1122          */
1123         talloc_free_children(this);
1124
1125         DEBUG("%s: Reconnecting (%" PRIu64 ")", pool->log_prefix, conn_number);
1126
1127         /*
1128          *      Allocate a new top level ctx for the create callback
1129          *      to hang its memory off of.
1130          */
1131         ctx = talloc_init("fr_connection_ctx");
1132         if (!ctx) return NULL;
1133         fr_link_talloc_ctx_free(this, ctx);
1134
1135         new_conn = pool->create(ctx, pool->opaque);
1136         if (!new_conn) {
1137                 /*
1138                  *      We can't create a new connection, so close
1139                  *      this one.
1140                  */
1141                 fr_connection_close(pool, this);
1142
1143                 /*
1144                  *      Maybe there's a connection which is unused and
1145                  *      available.  If so, return it.
1146                  */
1147                 pthread_mutex_unlock(&pool->mutex);
1148                 new_conn = fr_connection_get_internal(pool, false);
1149                 if (new_conn) return new_conn;
1150
1151                 RATE_LIMIT(ERROR("%s: Failed to reconnect (%" PRIu64 "), no free connections are available",
1152                                  pool->log_prefix, conn_number));
1153
1154                 return NULL;
1155         }
1156
1157         if (pool->trigger) exec_trigger(NULL, pool->cs, "close", true);
1158         pool->delete(pool->opaque, conn);
1159         this->connection = new_conn;
1160         pthread_mutex_unlock(&pool->mutex);
1161         return new_conn;
1162 }