Be more forgiving when starting proxy sockets.
[freeradius.git] / src / main / event.c
index 9a42dbe..844bdb6 100644 (file)
@@ -43,7 +43,6 @@ RCSID("$Id$")
 extern pid_t radius_pid;
 extern int dont_fork;
 extern int check_config;
-extern void force_log_reopen(void);
 extern char *debug_condition;
 
 /*
@@ -98,8 +97,11 @@ static fr_packet_list_t *proxy_list = NULL;
  *     We keep the proxy FD's here.  The RADIUS Id's are marked
  *     "allocated" per Id, via a bit per proxy FD.
  */
+static int             proxy_all_used = FALSE;
 static int             proxy_fds[32];
 static rad_listen_t    *proxy_listeners[32];
+static void check_for_zombie_home_server(REQUEST *request);
+static void remove_from_proxy_hash(REQUEST *request);
 #else
 #define remove_from_proxy_hash(foo)
 #endif
@@ -130,8 +132,8 @@ static void tv_add(struct timeval *tv, int usec_delay)
        tv->tv_usec += usec_delay;
 
        if (tv->tv_usec > USEC) {
-               tv->tv_usec -= USEC;
-               tv->tv_sec++;
+               tv->tv_sec += tv->tv_usec / USEC;
+               tv->tv_usec %= USEC;
        }
 }
 
@@ -146,6 +148,42 @@ static void remove_from_request_hash(REQUEST *request)
 }
 
 
+static void ev_request_free(REQUEST **prequest)
+{
+       REQUEST *request;
+       
+       if (!prequest || !*prequest) return;
+
+       request = *prequest;
+
+#ifdef WITH_COA
+       if (request->coa) {
+               /*
+                *      Divorce the child from the parent first,
+                *      then clean up the child.
+                */
+               request->coa->parent = NULL;
+               ev_request_free(&request->coa);
+       }
+
+       /*
+        *      Divorce the parent from the child, and leave the
+        *      parent still alive.
+        */
+       if (request->parent && (request->parent->coa == request)) {
+               request->parent->coa = NULL;
+       }
+#endif
+
+       if (request->ev) fr_event_delete(el, &request->ev);
+#ifdef WITH_PROXY
+       if (request->in_proxy_hash) remove_from_proxy_hash(request);
+#endif
+       if (request->in_request_hash) remove_from_request_hash(request);
+
+       request_free(prequest);
+}
+
 #ifdef WITH_PROXY
 static REQUEST *lookup_in_proxy_hash(RADIUS_PACKET *reply)
 {
@@ -241,47 +279,35 @@ static void remove_from_proxy_hash(REQUEST *request)
        PTHREAD_MUTEX_UNLOCK(&proxy_mutex);
 }
 
-static void ev_request_free(REQUEST **prequest)
+static int proxy_add_fds(rad_listen_t *proxy_listener)
 {
-       REQUEST *request;
-       
-       if (!prequest || !*prequest) return;
-
-       request = *prequest;
+       int i, proxy, found = -1;
 
-#ifdef WITH_COA
-       if (request->coa) {
+       proxy = proxy_listener->fd;
+       for (i = 0; i < 32; i++) {
                /*
-                *      Divorce the child from the parent first,
-                *      then clean up the child.
+                *      Found a free entry.  Save the socket,
+                *      and remember where we saved it.
                 */
-               request->coa->parent = NULL;
-               ev_request_free(&request->coa);
-       }
-
-       /*
-        *      Divorce the parent from the child, and leave the
-        *      parent still alive.
-        */
-       if (request->parent && (request->parent->coa == request)) {
-               request->parent->coa = NULL;
+               if (proxy_fds[(proxy + i) & 0x1f] == -1) {
+                       found = (proxy + i) & 0x1f;
+                       proxy_fds[found] = proxy;
+                       proxy_listeners[found] = proxy_listener;
+                       break;
+               }
        }
-#endif
-
-       if (request->ev) fr_event_delete(el, &request->ev);
-       if (request->in_proxy_hash) remove_from_proxy_hash(request);
-       if (request->in_request_hash) remove_from_request_hash(request);
 
-       request_free(prequest);
+       return found;
 }
 
 static int proxy_id_alloc(REQUEST *request, RADIUS_PACKET *packet)
 {
-       int i, proxy, found;
        rad_listen_t *proxy_listener;
 
        if (fr_packet_list_id_alloc(proxy_list, packet)) return 1;
 
+       if (proxy_all_used) return 0;
+
        /*
         *      Allocate a new proxy fd.  This function adds
         *      it to the tail of the list of listeners.  With
@@ -289,28 +315,20 @@ static int proxy_id_alloc(REQUEST *request, RADIUS_PACKET *packet)
         */
        proxy_listener = proxy_new_listener(&packet->src_ipaddr, FALSE);
        if (!proxy_listener) {
-               RDEBUG2("ERROR: Failed to create a new socket for proxying requests.");
+               radlog(L_PROXY, "Failed to create a new socket for proxying requests.");
                return 0;
        }
        
        /*
         *      Cache it locally.
         */
-       found = -1;
-       proxy = proxy_listener->fd;
-       for (i = 0; i < 32; i++) {
-               /*
-                *      Found a free entry.  Save the socket,
-                *      and remember where we saved it.
-                */
-               if (proxy_fds[(proxy + i) & 0x1f] == -1) {
-                       found = (proxy + i) & 0x1f;
-                       proxy_fds[found] = proxy;
-                       proxy_listeners[found] = proxy_listener;
-                       break;
-               }
+       if (proxy_add_fds(proxy_listener) < 0) {
+               proxy_all_used = TRUE;
+               listen_free(&proxy_listener);
+               radlog(L_ERR, "Failed creating new proxy socket: server is too busy and home servers appear to be down");
+               return 0;
        }
-       rad_assert(found >= 0);
+
        
        if (!fr_packet_list_socket_add(proxy_list, proxy_listener->fd)) {
                        RDEBUG2("ERROR: Failed to create a new socket for proxying requests.");
@@ -401,7 +419,7 @@ static int insert_into_proxy_hash(REQUEST *request, int retransmit)
        if (!fr_packet_list_insert(proxy_list, &request->proxy)) {
                fr_packet_list_id_free(proxy_list, request->proxy);
                PTHREAD_MUTEX_UNLOCK(&proxy_mutex);
-               RDEBUG2("ERROR: Failed to insert entry into proxy list");
+               RDEBUG2("ERROR: Failed to insert entry into proxy list.");
                return 0;
        }
 
@@ -444,11 +462,11 @@ static void wait_for_proxy_id_to_expire(void *ctx)
        if ((request->num_proxied_requests == request->num_proxied_responses) ||
            timercmp(&now, &request->when, >)) {
                if (request->packet) {
-                       RDEBUG2("Cleaning up request %d ID %d with timestamp +%d",
+                       RDEBUG2("Cleaning up request %u ID %d with timestamp +%d",
                               request->number, request->packet->id,
                               (unsigned int) (request->timestamp - fr_start_time));
                } else {
-                       RDEBUG2("Cleaning up request %d with timestamp +%d",
+                       RDEBUG2("Cleaning up request %u with timestamp +%d",
                               request->number,
                               (unsigned int) (request->timestamp - fr_start_time));
                }
@@ -467,20 +485,36 @@ static void wait_for_child_to_die(void *ctx)
        REQUEST *request = ctx;
 
        rad_assert(request->magic == REQUEST_MAGIC);
+       remove_from_request_hash(request);
 
-       if ((request->child_state == REQUEST_QUEUED) |
-           (request->child_state == REQUEST_RUNNING)) {
-               request->delay += (request->delay >> 1);
-               tv_add(&request->when, request->delay);
+       /*
+        *      If it's still queued (waiting for a thread to pick it
+        *      up) OR, it's running AND there's still a child thread
+        *      handling it, THEN delay some more.
+        */
+       if ((request->child_state == REQUEST_QUEUED) ||
+           ((request->child_state == REQUEST_RUNNING) &&
+            (pthread_equal(request->child_pid, NO_SUCH_CHILD_PID) == 0))) {
 
-               RDEBUG2("Child is still stuck for request %d", request->number);
+               /*
+                *      Cap delay at max_request_time
+                */
+               if (request->delay < (USEC * request->root->max_request_time)) {
+                       request->delay += (request->delay >> 1);
+                       radlog(L_INFO, "WARNING: Child is hung for request %u in component %s module %s.",
+                              request->number, request->component, request->module);
+               } else {
+                       request->delay = USEC * request->root->max_request_time;
+                       RDEBUG2("WARNING: Child is hung after \"max_request_time\" for request %u",
+                               request->number);
+               }
+               tv_add(&request->when, request->delay);
 
                INSERT_EVENT(wait_for_child_to_die, request);
                return;
        }
 
-       RDEBUG2("Child is finally responsive for request %d", request->number);
-       remove_from_request_hash(request);
+       RDEBUG2("Child is finally responsive for request %u", request->number);
 
 #ifdef WITH_PROXY
        if (request->proxy) {
@@ -510,7 +544,7 @@ static void cleanup_delay(void *ctx)
        }
 #endif
 
-       RDEBUG2("Cleaning up request %d ID %d with timestamp +%d",
+       RDEBUG2("Cleaning up request %u ID %d with timestamp +%d",
               request->number, request->packet->id,
               (unsigned int) (request->timestamp - fr_start_time));
 
@@ -519,66 +553,6 @@ static void cleanup_delay(void *ctx)
 
 
 /*
- *     FIXME: Put into a libradius function.
- */
-#define MAX_PACKET_CODE (52)
-static const char *packet_codes[] = {
-  "",
-  "Access-Request",
-  "Access-Accept",
-  "Access-Reject",
-  "Accounting-Request",
-  "Accounting-Response",
-  "Accounting-Status",
-  "Password-Request",
-  "Password-Accept",
-  "Password-Reject",
-  "Accounting-Message",
-  "Access-Challenge",
-  "Status-Server",
-  "Status-Client",
-  "14",
-  "15",
-  "16",
-  "17",
-  "18",
-  "19",
-  "20",
-  "Resource-Free-Request",
-  "Resource-Free-Response",
-  "Resource-Query-Request",
-  "Resource-Query-Response",
-  "Alternate-Resource-Reclaim-Request",
-  "NAS-Reboot-Request",
-  "NAS-Reboot-Response",
-  "28",
-  "Next-Passcode",
-  "New-Pin",
-  "Terminate-Session",
-  "Password-Expired",
-  "Event-Request",
-  "Event-Response",
-  "35",
-  "36",
-  "37",
-  "38",
-  "39",
-  "Disconnect-Request",
-  "Disconnect-ACK",
-  "Disconnect-NAK",
-  "CoA-Request",
-  "CoA-ACK",
-  "CoA-NAK",
-  "46",
-  "47",
-  "48",
-  "49",
-  "IP-Address-Allocate",
-  "IP-Address-Release"
-};
-
-
-/*
  *     In daemon mode, AND this request has debug flags set.
  */
 #define DEBUG_PACKET if (!debug_flag && request->options && request->radlog) debug_packet
@@ -614,9 +588,9 @@ static void debug_packet(REQUEST *request, RADIUS_PACKET *packet, int direction)
         *
         *      This really belongs in a utility library
         */
-       if ((packet->code > 0) && (packet->code < MAX_PACKET_CODE)) {
+       if ((packet->code > 0) && (packet->code < FR_MAX_PACKET_CODE)) {
                RDEBUG("%s %s packet %s host %s port %d, id=%d, length=%d",
-                      received, packet_codes[packet->code], from,
+                      received, fr_packet_codes[packet->code], from,
                       inet_ntop(ip->af, &ip->ipaddr, buffer, sizeof(buffer)),
                       port, packet->id, packet->data_len);
        } else {
@@ -640,7 +614,7 @@ static void reject_delay(void *ctx)
        rad_assert(request->magic == REQUEST_MAGIC);
        rad_assert(request->child_state == REQUEST_REJECT_DELAY);
 
-       RDEBUG2("Sending delayed reject for request %d", request->number);
+       RDEBUG2("Sending delayed reject for request %u", request->number);
 
        DEBUG_PACKET(request, request->reply, 1);
 
@@ -668,7 +642,7 @@ void revive_home_server(void *ctx)
         */
        if (home->ev) fr_event_delete(el, &home->ev);
 
-       radlog(L_INFO, "PROXY: Marking home server %s port %d alive again... we have no idea if it really is alive or not.",
+       radlog(L_PROXY, "Marking home server %s port %d alive again... we have no idea if it really is alive or not.",
               inet_ntop(home->ipaddr.af, &home->ipaddr.ipaddr,
                         buffer, sizeof(buffer)),
               home->port);
@@ -687,13 +661,15 @@ static void no_response_to_ping(void *ctx)
        home = request->home_server;
        home->num_received_pings = 0;
 
-       RDEBUG2("No response to status check %d from home server %s port %d",
+       radlog(L_ERR, "No response to status check %d for home server %s port %d",
               request->number,
               inet_ntop(request->proxy->dst_ipaddr.af,
                         &request->proxy->dst_ipaddr.ipaddr,
                         buffer, sizeof(buffer)),
               request->proxy->dst_port);
 
+       check_for_zombie_home_server(request);
+
        wait_for_proxy_id_to_expire(request);
 }
 
@@ -708,7 +684,7 @@ static void received_response_to_ping(REQUEST *request)
        home = request->home_server;
        home->num_received_pings++;
 
-       RDEBUG2("Received response to status check %d (%d in current sequence)",
+       radlog(L_PROXY, "Received response to status check %d (%d in current sequence)",
               request->number, home->num_received_pings);
 
        /*
@@ -742,7 +718,7 @@ static void received_response_to_ping(REQUEST *request)
                RDEBUG2("Hmm... no event for home server.  Oh well.");
        }
 
-       radlog(L_INFO, "PROXY: Marking home server %s port %d alive",
+       radlog(L_PROXY, "Marking home server %s port %d alive",
               inet_ntop(request->proxy->dst_ipaddr.af,
                         &request->proxy->dst_ipaddr.ipaddr,
                         buffer, sizeof(buffer)),
@@ -762,7 +738,8 @@ static void ping_home_server(void *ctx)
        VALUE_PAIR *vp;
 
        if ((home->state == HOME_STATE_ALIVE) ||
-           (home->ping_check == HOME_PING_CHECK_NONE)) {
+           (home->ping_check == HOME_PING_CHECK_NONE) ||
+           (home->ev != NULL)) {
                return;
        }
 
@@ -822,7 +799,7 @@ static void ping_home_server(void *ctx)
        rad_assert(request->proxy_listener == NULL);
 
        if (!insert_into_proxy_hash(request, FALSE)) {
-               RDEBUG2("ERROR: Failed inserting status check %d into proxy hash.  Discarding it.",
+               RDEBUG2("ERROR: Failed to insert status check %d into proxy list.  Discarding it.",
                       request->number);
                ev_request_free(&request);
                return;
@@ -833,6 +810,8 @@ static void ping_home_server(void *ctx)
 
        request->next_callback = NULL;
        request->child_state = REQUEST_PROXIED;
+       gettimeofday(&request->when, NULL);
+       home->when = request->when;
        request->when.tv_sec += home->ping_timeout;;
 
        INSERT_EVENT(no_response_to_ping, request);
@@ -858,7 +837,7 @@ void mark_home_server_dead(home_server *home, struct timeval *when)
        int previous_state = home->state;
        char buffer[128];
 
-       radlog(L_INFO, "PROXY: Marking home server %s port %d as dead.",
+       radlog(L_PROXY, "Marking home server %s port %d as dead.",
               inet_ntop(home->ipaddr.af, &home->ipaddr.ipaddr,
                         buffer, sizeof(buffer)),
               home->port);
@@ -1093,42 +1072,71 @@ static void no_response_to_proxied_request(void *ctx)
         *      well.
         */
        if (home->no_response_fail) {
-               radlog(L_ERR, "Rejecting request %d due to lack of any response from home server %s port %d",
-                      request->number,
+               radlog(L_ERR, "Rejecting request %u (proxy Id %d) due to lack of any response from home server %s port %d",
+                      request->number, request->proxy->id,
                       inet_ntop(request->proxy->dst_ipaddr.af,
                                 &request->proxy->dst_ipaddr.ipaddr,
                                 buffer, sizeof(buffer)),
                       request->proxy->dst_port);
 
                post_proxy_fail_handler(request);
+       } else {
+               rad_assert(request->ev == NULL);
+               request->child_state = REQUEST_RUNNING;
+               wait_a_bit(request);
        }
 
        /*
         *      Don't touch request due to race conditions
         */
-       if (home->state == HOME_STATE_IS_DEAD) {
-               rad_assert(home->ev != NULL); /* or it will never wake up */
-               return;
-       }
 
        /*
-        *      Enable the zombie period when we notice that the home
-        *      server hasn't responded.  We do NOT back-date the start
-        *      of the zombie period.
+        *      If it's not alive, don't try to make it a zombie.
         */
-       if (home->state == HOME_STATE_ALIVE) {
-               radlog(L_ERR, "PROXY: Marking home server %s port %d as zombie (it looks like it is dead).",
-                      inet_ntop(home->ipaddr.af, &home->ipaddr.ipaddr,
-                                buffer, sizeof(buffer)),
-                      home->port);
-               home->state = HOME_STATE_ZOMBIE;
-               home->zombie_period_start = now;
-
+       if (home->state != HOME_STATE_ALIVE) {
                /*
-                *      Start pinging the home server.
+                *      Don't check home->ev due to race conditions.
                 */
-               ping_home_server(home);
+               return;
        }
+
+       /*
+        *      We've received a real packet recently.  Don't mark the
+        *      server as zombie until we've received NO packets for a
+        *      while.  The "1/4" of zombie period was chosen rather
+        *      arbitrarily.  It's a balance between too short, which
+        *      gives quick fail-over and fail-back, or too long,
+        *      where the proxy still sends packets to an unresponsive
+        *      home server.
+        */
+       if ((home->last_packet + ((home->zombie_period + 3) / 4)) >= now.tv_sec) {
+               return;
+       }
+
+       /*
+        *      Enable the zombie period when we notice that the home
+        *      server hasn't responded for a while.  We back-date the
+        *      zombie period to when we last received a response from
+        *      the home server.
+        */
+       home->state = HOME_STATE_ZOMBIE;
+       
+       home->zombie_period_start.tv_sec = home->last_packet;
+       home->zombie_period_start.tv_sec = USEC / 2;
+       
+       fr_event_delete(el, &home->ev);
+       home->currently_outstanding = 0;
+       home->num_received_pings = 0;
+       
+       radlog(L_PROXY, "Marking home server %s port %d as zombie (it looks like it is dead).",
+              inet_ntop(home->ipaddr.af, &home->ipaddr.ipaddr,
+                        buffer, sizeof(buffer)),
+              home->port);
+       
+       /*
+        *      Start pinging the home server.
+        */
+       ping_home_server(home);
 }
 #endif
 
@@ -1160,6 +1168,25 @@ static void wait_a_bit(void *ctx)
        switch (request->child_state) {
        case REQUEST_QUEUED:
        case REQUEST_RUNNING:
+               /*
+                *      If we're not thread-capable, OR we're capable,
+                *      but have been told to run without threads,
+                *      complain when the requests is queued for a
+                *      thread, or running in a child thread.
+                */
+#ifdef HAVE_PTHREAD_H
+               if (!have_children)
+#endif
+               {
+                       rad_assert("We do not have threads, but the request is marked as queued or running in a child thread" == NULL);
+                       break;
+               }
+
+#ifdef HAVE_PTHREAD_H
+               /*
+                *      If we have threads, wait for the child thread
+                *      to stop.
+                */
                when = request->received;
                when.tv_sec += request->root->max_request_time;
 
@@ -1176,24 +1203,18 @@ static void wait_a_bit(void *ctx)
                 *      Request still has more time.  Continue
                 *      waiting.
                 */
-               if (timercmp(&now, &when, <) ||
-                   ((request->listener->type == RAD_LISTEN_DETAIL) &&
-                    (request->child_state == REQUEST_QUEUED))) {
+               if (timercmp(&now, &when, <)) {
                        if (request->delay < (USEC / 10)) {
                                request->delay = USEC / 10;
                        }
                        request->delay += request->delay >> 1;
 
-#ifdef WITH_DETAIL
                        /*
-                        *      Cap wait at some sane value for detail
-                        *      files.
+                        *      Cap delays at something reasonable.
                         */
-                       if ((request->listener->type == RAD_LISTEN_DETAIL) &&
-                           (request->delay > (request->root->max_request_time * USEC))) {
+                       if (request->delay > (request->root->max_request_time * USEC)) {
                                request->delay = request->root->max_request_time * USEC;
                        }
-#endif
 
                        request->when = now;
                        tv_add(&request->when, request->delay);
@@ -1201,51 +1222,29 @@ static void wait_a_bit(void *ctx)
                        break;
                }
 
-#if defined(HAVE_PTHREAD_H) || defined(WITH_PROXY)
+               request->master_state = REQUEST_STOP_PROCESSING;
+
                /*
                 *      A child thread MAY still be running on the
                 *      request.  Ask the thread to stop working on
                 *      the request.
                 */
-               if (have_children) {
-                       /* FIXME: kill unresponsive children? */
-
-                       /*
-                        *      Print this error message ONLY if
-                        *      there's a child currently processing
-                        *      the request.  As we don't have thread
-                        *      locks here, there may be race
-                        *      conditions on this check.  But it's
-                        *      just an error message, so that's OK.
-                        */
-                       if (!pthread_equal(request->child_pid, NO_SUCH_CHILD_PID)) {
-                               radlog(L_ERR, "WARNING: Unresponsive child for request %d, in module %s component %s",
-                                      request->number,
-                                      request->module ? request->module : "<server core>",
-                                      request->component ? request->component : "<server core>");
-                       }
+               if (have_children &&
+                   (pthread_equal(request->child_pid, NO_SUCH_CHILD_PID) == 0)) {
+                       radlog(L_ERR, "WARNING: Unresponsive child for request %u, in component %s module %s",
+                              request->number,
+                              request->component ? request->component : "<server core>",
+                              request->module ? request->module : "<server core>");
 
-                       request->master_state = REQUEST_STOP_PROCESSING;
-                       
-                       request->delay = USEC / 4;
-                       tv_add(&request->when, request->delay);
-                       callback = wait_for_child_to_die;
-                       break;
                }
+                       
+               request->delay = USEC;
+               tv_add(&request->when, request->delay);
+               callback = wait_for_child_to_die;
+               break;
 #endif
 
                /*
-                *      Else there are no child threads.  We probably
-                *      should have just marked the request as 'done'
-                *      elsewhere, like in the post-proxy-fail
-                *      handler.  But doing that would involve
-                *      checking for max_request_time in multiple
-                *      places, so this may be simplest.
-                */
-               request->child_state = REQUEST_DONE;
-               /* FALL-THROUGH */
-
-               /*
                 *      Mark the request as no longer running,
                 *      and clean it up.
                 */
@@ -1254,7 +1253,7 @@ static void wait_a_bit(void *ctx)
                request->child_pid = NO_SUCH_CHILD_PID;
 #endif
 
-#ifdef WTH_COA
+#ifdef WITH_COA
                /*
                 *      This is a CoA request.  It's been divorced
                 *      from everything else, so we clean it up now.
@@ -1307,7 +1306,7 @@ static void wait_a_bit(void *ctx)
         *      mode, with no threads...
         */
        if (!callback) {
-               RDEBUG("WARNING: Internal sanity check failed in event handler for request %d: Discarding the request!", request->number);
+               RDEBUG("WARNING: Internal sanity check failed in event handler for request %u: Discarding the request!", request->number);
                ev_request_free(&request);
                return;
        }
@@ -1436,7 +1435,7 @@ static void retransmit_coa_request(void *ctx)
        
        if (update_event_timestamp(request->proxy, now.tv_sec)) {
                if (!insert_into_proxy_hash(request, TRUE)) {
-                       DEBUG("ERROR: Failed re-inserting CoA request into proxy hash.");
+                       DEBUG("ERROR: Failed to insert retransmission of CoA request into proxy list.");
                        return;
                }
 
@@ -1467,18 +1466,24 @@ static int originated_coa_request(REQUEST *request)
        rad_assert(!request->in_proxy_hash);
        rad_assert(request->proxy_reply == NULL);
 
+       /*
+        *      Check whether we want to originate one, or cancel one.
+        */
        vp = pairfind(request->config_items, PW_SEND_COA_REQUEST);
-       if (!vp && request->coa) vp = pairfind(request->coa->proxy->vps, PW_SEND_COA_REQUEST);
+       if (!vp && request->coa) {
+               vp = pairfind(request->coa->proxy->vps, PW_SEND_COA_REQUEST);
+       }
+
        if (vp) {
                if (vp->vp_integer == 0) {
                        ev_request_free(&request->coa);
                        return 1;       /* success */
                }
-
-               if (!request->coa) request_alloc_coa(request);
-               if (!request->coa) return 0;
        }
 
+       if (!request->coa) request_alloc_coa(request);
+       if (!request->coa) return 0;
+
        coa = request->coa;
 
        /*
@@ -1624,7 +1629,7 @@ static int originated_coa_request(REQUEST *request)
        coa->proxy->dst_port = coa->home_server->port;
 
        if (!insert_into_proxy_hash(coa, FALSE)) {
-               DEBUG("ERROR: Failed inserting CoA request into proxy hash.");
+               DEBUG("ERROR: Failed to insert CoA request into proxy list.");
                goto fail;
        }
 
@@ -1675,7 +1680,9 @@ static int originated_coa_request(REQUEST *request)
         */
        request->num_proxied_requests = 1;
        request->num_proxied_responses = 0;
+#ifdef HAVE_PTHREAD_H
        request->child_pid = NO_SUCH_CHILD_PID;
+#endif
 
        update_event_timestamp(request->proxy, request->proxy_when.tv_sec);
 
@@ -1832,7 +1839,8 @@ static int request_pre_handler(REQUEST *request)
        }
 
        if (rcode < 0) {
-               radlog(L_ERR, "%s Dropping packet without response.", fr_strerror());
+               RDEBUG("%s Dropping packet without response.", fr_strerror());
+               request->reply->offset = -2; /* bad authenticator */
                request->child_state = REQUEST_DONE;
                return 0;
        }
@@ -1845,8 +1853,8 @@ static int request_pre_handler(REQUEST *request)
 #ifdef WITH_PROXY
        if (request->proxy) {
                return process_proxy_reply(request);
-#endif
        }
+#endif
 
        return 1;
 }
@@ -1861,13 +1869,20 @@ static int proxy_request(REQUEST *request)
        struct timeval when;
        char buffer[128];
 
+#ifdef WITH_COA
+       if (request->coa) {
+               RDEBUG("WARNING: Cannot proxy and originate CoA packets at the same time.  Cancelling CoA request");
+               ev_request_free(&request->coa);
+       }
+#endif
+
        if (request->home_server->server) {
-               RDEBUG("ERROR: Cannot perform real proxying to a virtual server.");
+               RDEBUG("ERROR: Cannot proxy to a virtual server.");
                return 0;
        }
 
        if (!insert_into_proxy_hash(request, FALSE)) {
-               RDEBUG("ERROR: Failed inserting request into proxy hash.");
+               RDEBUG("ERROR: Failed to insert entry into proxy list.");
                return 0;
        }
 
@@ -1888,7 +1903,7 @@ static int proxy_request(REQUEST *request)
        }
        request->next_callback = no_response_to_proxied_request;
 
-       RDEBUG2("Proxying request %d to home server %s port %d",
+       RDEBUG2("Proxying request %u to home server %s port %d",
               request->number,
               inet_ntop(request->proxy->dst_ipaddr.af,
                         &request->proxy->dst_ipaddr.ipaddr,
@@ -2245,7 +2260,7 @@ found_pool:
        }
 
        if (!proxy_request(request)) {
-               RDEBUG("ERROR: Failed to proxy request %d", request->number);
+               RDEBUG("ERROR: Failed to proxy request %u", request->number);
                return -1;
        }
        
@@ -2262,7 +2277,7 @@ static void request_post_handler(REQUEST *request)
        if ((request->master_state == REQUEST_STOP_PROCESSING) ||
            (request->parent &&
             (request->parent->master_state == REQUEST_STOP_PROCESSING))) {
-               RDEBUG2("Request %d was cancelled.", request->number);
+               RDEBUG2("request %u was cancelled.", request->number);
 #ifdef HAVE_PTHREAD_H
                request->child_pid = NO_SUCH_CHILD_PID;
 #endif
@@ -2289,7 +2304,11 @@ static void request_post_handler(REQUEST *request)
        }
 #endif
 
-       if ((request->reply->code == 0) &&
+       /*
+        *      Catch Auth-Type := Reject BEFORE proxying the packet.
+        */
+       if ((request->packet->code == PW_AUTHENTICATION_REQUEST) &&
+           (request->reply->code == 0) &&
            ((vp = pairfind(request->config_items, PW_AUTH_TYPE)) != NULL) &&
            (vp->vp_integer == PW_AUTHTYPE_REJECT)) {
                request->reply->code = PW_AUTHENTICATION_REJECT;
@@ -2325,6 +2344,13 @@ static void request_post_handler(REQUEST *request)
                 *      OR we proxied it internally to a virutal server.
                 */
        }
+
+#ifdef WITH_COA
+       else if (request->proxy && request->coa) {
+               RDEBUG("WARNING: Cannot proxy and originate CoA packets at the same time.  Cancelling CoA request");
+               ev_request_free(&request->coa);
+       }
+#endif
 #endif
 
        /*
@@ -2363,12 +2389,12 @@ static void request_post_handler(REQUEST *request)
                        vp = pairfind(request->config_items,
                                      PW_RESPONSE_PACKET_TYPE);
                        if (!vp) {
-                               RDEBUG2("There was no response configured: rejecting request %d",
+                               RDEBUG2("There was no response configured: rejecting request %u",
                                       request->number);
                                request->reply->code = PW_AUTHENTICATION_REJECT;
 
                        } else if (vp->vp_integer == 256) {
-                               RDEBUG2("Not responding to request %d",
+                               RDEBUG2("Not responding to request %u",
                                       request->number);
 
                                /*
@@ -2408,7 +2434,7 @@ static void request_post_handler(REQUEST *request)
                        when.tv_sec += request->root->reject_delay;
 
                        if (timercmp(&when, &request->next_when, >)) {
-                               RDEBUG2("Delaying reject of request %d for %d seconds",
+                               RDEBUG2("Delaying reject of request %u for %d seconds",
                                       request->number,
                                       request->root->reject_delay);
                                request->next_when = when;
@@ -2445,14 +2471,9 @@ static void request_post_handler(REQUEST *request)
                break;
 
        default:
-               if ((request->packet->code > 1024) &&
-                   (request->packet->code < (1024 + 254 + 1))) {
-                       request->next_callback = NULL;
-                       child_state = REQUEST_DONE;
-                       break;
-               }
-
-               radlog(L_ERR, "Unknown packet type %d", request->packet->code);
+               /*
+                *      DHCP, VMPS, etc.
+                */
                request->next_callback = NULL;
                child_state = REQUEST_DONE;
                break;
@@ -2474,10 +2495,14 @@ static void request_post_handler(REQUEST *request)
 #ifdef WITH_COA
        /*
         *      Now that we've completely processed the request,
-        *      see if we need to originate a CoA request.
-        */
-       if (request->coa ||
-           (pairfind(request->config_items, PW_SEND_COA_REQUEST) != NULL)) {
+        *      see if we need to originate a CoA request.  But ONLY
+        *      if it wasn't proxied.
+        */
+       if (!request->proxy &&
+           (request->packet->code != PW_COA_REQUEST) &&
+           (request->packet->code != PW_DISCONNECT_REQUEST) &&
+           (request->coa ||
+            (pairfind(request->config_items, PW_SEND_COA_REQUEST) != NULL))) {
                if (!originated_coa_request(request)) {
                        RDEBUG2("Do CoA Fail handler here");
                }
@@ -2520,7 +2545,7 @@ static void request_post_handler(REQUEST *request)
        }
 #endif
 
-       RDEBUG2("Finished request %d.", request->number);
+       RDEBUG2("Finished request %u.", request->number);
        rad_assert(child_state >= 0);
        request->child_state = child_state;
 
@@ -2547,7 +2572,7 @@ static void received_retransmit(REQUEST *request, const RADCLIENT *client)
        discard:
 #endif
                radlog(L_ERR, "Discarding duplicate request from "
-                      "client %s port %d - ID: %d due to unfinished request %d",
+                      "client %s port %d - ID: %d due to unfinished request %u",
                       client->shortname,
                       request->packet->src_port,request->packet->id,
                       request->number);
@@ -2592,7 +2617,7 @@ static void received_retransmit(REQUEST *request, const RADCLIENT *client)
 
                        home = home_server_ldb(NULL, request->home_pool, request);
                        if (!home) {
-                               RDEBUG2("Failed to find live home server for request %d", request->number);
+                               RDEBUG2("ERROR: Failed to find live home server for request %u", request->number);
                        no_home_servers:
                                /*
                                 *      Do post-request processing,
@@ -2626,7 +2651,7 @@ static void received_retransmit(REQUEST *request, const RADCLIENT *client)
                         *      Try to proxy the request.
                         */
                        if (!proxy_request(request)) {
-                               RDEBUG("ERROR: Failed to re-proxy request %d", request->number);
+                               RDEBUG("ERROR: Failed to re-proxy request %u", request->number);
                                goto no_home_servers;
                        }
 
@@ -2693,7 +2718,7 @@ static void received_conflicting_request(REQUEST *request,
                                         const RADCLIENT *client)
 {
        radlog(L_ERR, "Received conflicting packet from "
-              "client %s port %d - ID: %d due to unfinished request %d.  Giving up on old request.",
+              "client %s port %d - ID: %d due to unfinished request %u.  Giving up on old request.",
               client->shortname,
               request->packet->src_port, request->packet->id,
               request->number);
@@ -2833,7 +2858,7 @@ int received_request(rad_listen_t *listener,
                         */
                        if ((request->reply->code != 0) &&
                            request->reply->data) {
-                               radlog(L_INFO, "WARNING: Allowing fast client %s port %d - ID: %d for recent request %d.",
+                               radlog(L_INFO, "WARNING: Allowing fast client %s port %d - ID: %d for recent request %u.",
                                       client->shortname,
                                       packet->src_port, packet->id,
                                       request->number);
@@ -2856,7 +2881,7 @@ int received_request(rad_listen_t *listener,
                         */
                        if (timercmp(&when, &request->received, <)) {
                                radlog(L_ERR, "Discarding conflicting packet from "
-                                      "client %s port %d - ID: %d due to recent request %d.",
+                                      "client %s port %d - ID: %d due to recent request %u.",
                                       client->shortname,
                                       packet->src_port, packet->id,
                                       request->number);
@@ -2925,7 +2950,7 @@ int received_request(rad_listen_t *listener,
         *      Remember the request in the list.
         */
        if (!fr_packet_list_insert(pl, &request->packet)) {
-               radlog(L_ERR, "Failed to insert request %d in the list of live requests: discarding", request->number);
+               radlog(L_ERR, "Failed to insert request %u in the list of live requests: discarding", request->number);
                ev_request_free(&request);
                return 0;
        }
@@ -3006,7 +3031,7 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
                if (memcmp(request->proxy_reply->vector,
                           packet->vector,
                           sizeof(request->proxy_reply->vector)) == 0) {
-                       RDEBUG2("Discarding duplicate reply from host %s port %d  - ID: %d for request %d",
+                       RDEBUG2("Discarding duplicate reply from host %s port %d  - ID: %d for request %u",
                               inet_ntop(packet->src_ipaddr.af,
                                         &packet->src_ipaddr.ipaddr,
                                         buffer, sizeof(buffer)),
@@ -3045,6 +3070,19 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
        gettimeofday(&now, NULL);
 
        /*
+        *      "ping" packets have a different algorithm for marking
+        *      a home server alive.  They also skip all of the CoA,
+        *      etc. checks.
+        */
+       if (!request->packet) {
+               request->proxy_reply = packet;
+               received_response_to_ping(request);
+               request->proxy_reply = NULL; /* caller will free it */
+               ev_request_free(&request);
+               return NULL;
+       }
+
+       /*
         *      Maybe move this earlier in the decision process?
         *      Having it here means that late or duplicate proxy
         *      replies no longer get the home server marked as
@@ -3054,8 +3092,11 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
         *      receive a packet?  Setting this here means that we
         *      mark it alive on *any* packet, even if it's lost all
         *      of the *other* packets in the last 10s.
+        *
+        *      This behavior could be configurable.
         */
        request->home_server->state = HOME_STATE_ALIVE;
+       request->home_server->last_packet = now.tv_sec;
        
 #ifdef WITH_COA
        /*
@@ -3072,6 +3113,18 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
                request->parent->coa = NULL;
                request->parent = NULL;
 
+               /*
+                *      The proxied packet was different from the
+                *      original packet, AND the proxied packet was
+                *      a CoA: allow it.
+                */
+       } else if ((request->packet->code != request->proxy->code) &&
+                  ((request->proxy->code == PW_COA_REQUEST) ||
+                   (request->proxy->code == PW_DISCONNECT_REQUEST))) {
+         /*
+          *    It's already divorced: do nothing.
+          */
+         
        } else
                /*
                 *      Skip the next set of checks, as the original
@@ -3092,7 +3145,7 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
                RDEBUG2("Ignoring proxy reply that arrived after we sent a reply to the NAS");
                return NULL;
        }
-       
+
 #ifdef WITH_STATS
        /*
         *      The average includes our time to receive packets and
@@ -3117,7 +3170,7 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
        case REQUEST_REJECT_DELAY:
        case REQUEST_CLEANUP_DELAY:
        case REQUEST_DONE:
-               radlog(L_ERR, "Reply from home server %s port %d  - ID: %d arrived too late for request %d. Try increasing 'retry_delay' or 'max_request_time'",
+               radlog(L_ERR, "Reply from home server %s port %d  - ID: %d arrived too late for request %u. Try increasing 'retry_delay' or 'max_request_time'",
                       inet_ntop(packet->src_ipaddr.af,
                                 &packet->src_ipaddr.ipaddr,
                                 buffer, sizeof(buffer)),
@@ -3168,17 +3221,6 @@ REQUEST *received_proxy_response(RADIUS_PACKET *packet)
        }
 #endif
 
-       /*
-        *      There's no incoming request, so it's a proxied packet
-        *      we originated.
-        */
-       if (!request->packet) {
-               received_response_to_ping(request);
-               request->proxy_reply = NULL; /* caller will free it */
-               ev_request_free(&request);
-               return NULL;
-       }
-
        request->child_state = REQUEST_QUEUED;
        request->when = now;
        request->delay = USEC;
@@ -3206,7 +3248,7 @@ void event_new_fd(rad_listen_t *this)
                if (just_started) {
                        DEBUG("Listening on %s", buffer);
                } else {
-                       DEBUG2(" ... adding new socket %s", buffer);
+                       radlog(L_INFO, " ... adding new socket %s", buffer);
                }
                if (!fr_event_fd_insert(el, 0, this->fd,
                                        event_socket_handler, this)) {
@@ -3219,7 +3261,7 @@ void event_new_fd(rad_listen_t *this)
        }
        
        if (this->status == RAD_LISTEN_STATUS_CLOSED) {
-               DEBUG2(" ... closing socket %s", buffer);
+               radlog(L_INFO, " ... closing socket %s", buffer);
                
                fr_event_fd_delete(el, 0, this->fd);
                this->status = RAD_LISTEN_STATUS_FINISH;
@@ -3254,13 +3296,14 @@ static void handle_signal_self(int flag)
                time_t when;
                static time_t last_hup = 0;
 
-               DEBUG("Received HUP signal.");
-
                when = time(NULL);
                if ((int) (when - last_hup) < 5) {
                        radlog(L_INFO, "Ignoring HUP (less than 5s since last one)");
                        return;
                }
+
+               radlog(L_INFO, "Received HUP signal.");
+
                last_hup = when;
 
                fr_event_loop_exit(el, 0x80);
@@ -3378,9 +3421,6 @@ static void event_socket_handler(fr_event_list_t *xel, UNUSED int fd,
        }
 }
 
-typedef struct listen_detail_t {
-       fr_event_t      *ev;
-} listen_detail_t;
 
 /*
  *     This function is called periodically to see if this detail
@@ -3445,7 +3485,7 @@ static void event_status(struct timeval *wake)
        }
 
        if (!wake) {
-               DEBUG("Ready to process requests.");
+               radlog(L_INFO, "Ready to process requests.");
 
        } else if ((wake->tv_sec != 0) ||
                   (wake->tv_usec >= 100000)) {
@@ -3494,8 +3534,6 @@ int radius_event_init(CONF_SECTION *cs, int spawn_flag)
 
 #ifdef WITH_PROXY
        if (mainconfig.proxy_requests) {
-               pthread_mutexattr_t attr;
-
                /*
                 *      Create the tree for managing proxied requests and
                 *      responses.
@@ -3504,33 +3542,15 @@ int radius_event_init(CONF_SECTION *cs, int spawn_flag)
                if (!proxy_list) return 0;
 
 #ifdef HAVE_PTHREAD_H
-               pthread_mutexattr_init(&attr);
-
-#ifdef PTHREAD_MUTEX_RECURSIVE
-               if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE) < 0) {
-                       radlog(L_ERR, "FATAL: Failed to set type for proxy mutex: %s",
-                              strerror(errno));
-                       exit(1);
-               }
-#endif
-
                if (pthread_mutex_init(&proxy_mutex, NULL) != 0) {
                        radlog(L_ERR, "FATAL: Failed to initialize proxy mutex: %s",
                               strerror(errno));
                        exit(1);
                }
-
-               pthread_mutexattr_destroy(&attr);
 #endif
        }
 #endif
 
-       /*
-        *      Just before we spawn the child threads, force the log
-        *      subsystem to re-open the log file for every write.
-        */
-       if (spawn_flag) force_log_reopen();
-
 #ifdef HAVE_PTHREAD_H
 #ifndef __MINGW32__
        NO_SUCH_CHILD_PID = (pthread_t ) (0);
@@ -3557,6 +3577,10 @@ int radius_event_init(CONF_SECTION *cs, int spawn_flag)
        if (check_config) {
                DEBUG("%s: #### Skipping IP addresses and Ports ####",
                       mainconfig.name);
+               if (listen_init(cs, &head) < 0) {
+                       fflush(NULL);
+                       exit(1);
+               }
                return 1;
        }
 
@@ -3651,11 +3675,11 @@ int radius_event_init(CONF_SECTION *cs, int spawn_flag)
 
 #ifdef WITH_PROXY
                case RAD_LISTEN_PROXY:
-                       rad_assert(proxy_fds[this->fd & 0x1f] == -1);
-                       rad_assert(proxy_listeners[this->fd & 0x1f] == NULL);
-                       
-                       proxy_fds[this->fd & 0x1f] = this->fd;
-                       proxy_listeners[this->fd & 0x1f] = this;
+                       if (proxy_add_fds(this) < 0) {
+                               radlog(L_ERR, "Failed creating new proxy socket");
+                               return 0;
+                       }
+
                        if (!fr_packet_list_socket_add(proxy_list,
                                                         this->fd)) {
                                rad_assert(0 == 1);
@@ -3715,9 +3739,7 @@ void radius_event_free(void)
         *      referenced from anywhere else.  Remove them first.
         */
        if (proxy_list) {
-               PTHREAD_MUTEX_LOCK(&proxy_mutex);
                fr_packet_list_walk(proxy_list, NULL, proxy_hash_cb);
-               PTHREAD_MUTEX_UNLOCK(&proxy_mutex);
                fr_packet_list_free(proxy_list);
                proxy_list = NULL;
        }