2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
19 * @brief Various functions to aid in debugging
21 * @copyright 2013 The FreeRADIUS server project
22 * @copyright 2013 Arran Cudbard-Bell <a.cudbardb@freeradius.org>
25 #include <freeradius-devel/libradius.h>
28 #if defined(HAVE_MALLOPT) && defined(HAVE_MALLOC_H)
33 * runtime backtrace functions are not POSIX but are included in
34 * glibc, OSX >= 10.5 and various BSDs
37 # include <execinfo.h>
40 #ifdef HAVE_SYS_PRCTL_H
41 # include <sys/prctl.h>
44 #ifdef HAVE_SYS_RESOURCE_H
45 # include <sys/resource.h>
49 # define PTHREAD_MUTEX_LOCK pthread_mutex_lock
50 # define PTHREAD_MUTEX_UNLOCK pthread_mutex_unlock
52 # define PTHREAD_MUTEX_LOCK(_x)
53 # define PTHREAD_MUTEX_UNLOCK(_x)
57 # define MAX_BT_FRAMES 128
58 # define MAX_BT_CBUFF 65536 //!< Should be a power of 2
60 # ifdef HAVE_PTHREAD_H
61 static pthread_mutex_t fr_debug_init = PTHREAD_MUTEX_INITIALIZER;
64 typedef struct fr_bt_info {
65 void *obj; //!< Memory address of the block of allocated memory.
66 void *frames[MAX_BT_FRAMES]; //!< Backtrace frame data
67 int count; //!< Number of frames stored
71 void *obj; //!< Pointer to the parent object, this is our needle
72 //!< when we iterate over the contents of the circular buffer.
73 fr_cbuff_t *cbuff; //!< Where we temporarily store the backtraces
77 static char panic_action[512]; //!< The command to execute when panicking.
78 static fr_fault_cb_t panic_cb = NULL; //!< Callback to execute whilst panicking, before the
80 static fr_fault_log_t fr_fault_log = NULL; //!< Function to use to process logging output.
81 static int fr_fault_log_fd = STDERR_FILENO; //!< Where to write debug output.
83 static int fr_debugger_present = -1; //!< Whether were attached to by a debugger.
85 #ifdef HAVE_SYS_RESOURCE_H
86 static struct rlimit core_limits;
89 static TALLOC_CTX *talloc_null_ctx;
90 static TALLOC_CTX *talloc_autofree_ctx;
92 #define FR_FAULT_LOG(fmt, ...) fr_fault_log(fmt "\n", ## __VA_ARGS__)
94 /** Stub callback to see if the SIGTRAP handler is overriden
96 * @param signum signal raised.
98 static void _sigtrap_handler(UNUSED int signum)
100 fr_debugger_present = 0;
101 signal(SIGTRAP, SIG_DFL);
104 /** Break in debugger (if were running under a debugger)
106 * If the server is running under a debugger this will raise a
107 * SIGTRAP which will pause the running process.
109 * If the server is not running under debugger then this will do nothing.
111 void fr_debug_break(void)
113 if (fr_debugger_present == -1) {
114 fr_debugger_present = 0;
115 signal(SIGTRAP, _sigtrap_handler);
117 } else if (fr_debugger_present == 1) {
123 /** Print backtrace entry for a given object
125 * @param cbuff to search in.
126 * @param obj pointer to original object
128 void backtrace_print(fr_cbuff_t *cbuff, void *obj)
133 while ((p = fr_cbuff_rp_next(cbuff, NULL))) {
134 if ((p->obj == obj) || !obj) {
137 fprintf(stderr, "Stacktrace for: %p\n", p->obj);
138 backtrace_symbols_fd(p->frames, p->count, STDERR_FILENO);
143 fprintf(stderr, "No backtrace available for %p", obj);
147 /** Generate a backtrace for an object
149 * If this is the first entry being inserted
151 int fr_backtrace_do(fr_bt_marker_t *marker)
155 if (!fr_assert(marker->obj) || !fr_assert(marker->cbuff)) return -1;
157 bt = talloc_zero(NULL, fr_bt_info_t);
160 bt->obj = marker->obj;
161 bt->count = backtrace(bt->frames, MAX_BT_FRAMES);
163 fr_cbuff_rp_insert(marker->cbuff, bt);
168 /** Inserts a backtrace marker into the provided context
170 * Allows for maximum laziness and will initialise a circular buffer if one has not already been created.
172 * Code augmentation should look something like:
174 // Create a static cbuffer pointer, the first call to backtrace_attach will initialise it
175 static fr_cbuff_t *my_obj_bt;
177 my_obj_t *alloc_my_obj(TALLOC_CTX *ctx) {
180 this = talloc(ctx, my_obj_t);
182 // Attach backtrace marker to object
183 backtrace_attach(&my_obj_bt, this);
189 * Then, later when a double free occurs:
191 (gdb) call backtrace_print(&my_obj_bt, <pointer to double freed memory>)
194 * which should print a limited backtrace to stderr. Note, this backtrace will not include any argument
195 * values, but should at least show the code path taken.
197 * @param cbuff this should be a pointer to a static *fr_cbuff.
198 * @param obj we want to generate a backtrace for.
200 fr_bt_marker_t *fr_backtrace_attach(fr_cbuff_t **cbuff, TALLOC_CTX *obj)
202 fr_bt_marker_t *marker;
204 if (*cbuff == NULL) {
205 PTHREAD_MUTEX_LOCK(&fr_debug_init);
206 /* Check again now we hold the mutex - eww*/
207 if (*cbuff == NULL) *cbuff = fr_cbuff_alloc(NULL, MAX_BT_CBUFF, true);
208 PTHREAD_MUTEX_UNLOCK(&fr_debug_init);
211 marker = talloc(obj, fr_bt_marker_t);
216 marker->obj = (void *) obj;
217 marker->cbuff = *cbuff;
219 fprintf(stderr, "Backtrace attached to %s %p\n", talloc_get_name(obj), obj);
221 * Generate the backtrace for memory allocation
223 fr_backtrace_do(marker);
224 talloc_set_destructor(marker, fr_backtrace_do);
229 void backtrace_print(UNUSED fr_cbuff_t *cbuff, UNUSED void *obj)
231 fprintf(stderr, "Server built without fr_backtrace_* support, requires execinfo.h and possibly -lexecinfo\n");
233 fr_bt_marker_t *fr_backtrace_attach(UNUSED fr_cbuff_t **cbuff, UNUSED TALLOC_CTX *obj)
235 fprintf(stderr, "Server built without fr_backtrace_* support, requires execinfo.h and possibly -lexecinfo\n");
238 #endif /* ifdef HAVE_EXECINFO */
240 static int _panic_on_free(UNUSED char *foo)
243 return -1; /* this should make the free fail */
246 /** Insert memory into the context of another talloc memory chunk which
247 * causes a panic when freed.
249 * @param ctx TALLOC_CTX to monitor for frees.
251 void fr_panic_on_free(TALLOC_CTX *ctx)
255 ptr = talloc(ctx, char);
256 talloc_set_destructor(ptr, _panic_on_free);
259 /** Set the dumpable flag, also controls whether processes can PATTACH
261 * @param dumpable whether we should allow core dumping
263 #if defined(HAVE_SYS_PRCTL_H) && defined(PR_SET_DUMPABLE)
264 static int fr_set_dumpable_flag(bool dumpable)
266 if (prctl(PR_SET_DUMPABLE, dumpable ? 1 : 0) < 0) {
267 fr_strerror_printf("Cannot re-enable core dumps: prctl(PR_SET_DUMPABLE) failed: %s",
275 static int fr_set_dumpable_flag(UNUSED bool dumpable)
277 fr_strerror_printf("Changing value of PR_DUMPABLE not supported on this system");
282 /** Get the processes dumpable flag
285 #if defined(HAVE_SYS_PRCTL_H) && defined(PR_GET_DUMPABLE)
286 static int fr_get_dumpable_flag(void)
290 ret = prctl(PR_GET_DUMPABLE);
292 fr_strerror_printf("Cannot get dumpable flag: %s", fr_syserror(errno));
297 * Linux is crazy and prctl sometimes returns 2 for disabled
299 if (ret != 1) return 0;
303 static int fr_get_dumpable_flag(void)
305 fr_strerror_printf("Getting value of PR_DUMPABLE not supported on this system");
311 /** Get the current maximum for core files
313 * Do this before anything else so as to ensure it's properly initialized.
315 int fr_set_dumpable_init(void)
317 #ifdef HAVE_SYS_RESOURCE_H
318 if (getrlimit(RLIMIT_CORE, &core_limits) < 0) {
319 fr_strerror_printf("Failed to get current core limit: %s", fr_syserror(errno));
326 /** Enable or disable core dumps
328 * @param allow_core_dumps whether to enable or disable core dumps.
330 int fr_set_dumpable(bool allow_core_dumps)
333 * If configured, turn core dumps off.
335 if (!allow_core_dumps) {
336 #ifdef HAVE_SYS_RESOURCE_H
337 struct rlimit no_core;
339 no_core.rlim_cur = 0;
340 no_core.rlim_max = 0;
342 if (setrlimit(RLIMIT_CORE, &no_core) < 0) {
343 fr_strerror_printf("Failed disabling core dumps: %s", fr_syserror(errno));
351 if (fr_set_dumpable_flag(true) < 0) return -1;
354 * Reset the core dump limits to their original value.
356 #ifdef HAVE_SYS_RESOURCE_H
357 if (setrlimit(RLIMIT_CORE, &core_limits) < 0) {
358 fr_strerror_printf("Cannot update core dump limit: %s", fr_syserror(errno));
366 /** Check to see if panic_action file is world writeable
368 * @return 0 if file is OK, else -1.
370 static int fr_fault_check_permissions(void)
378 * Try and guess which part of the command is the binary, and check to see if
379 * it's world writeable, to try and save the admin from their own stupidity.
381 * @fixme we should do this properly and take into account single and double
384 if ((q = strchr(panic_action, ' '))) {
386 * need to use a static buffer, because mallocing memory in a signal handler
387 * is a bad idea and can result in deadlock.
389 len = snprintf(filename, sizeof(filename), "%.*s", (int)(q - panic_action), panic_action);
390 if (is_truncated(len, sizeof(filename))) {
391 fr_strerror_printf("Failed writing panic_action to temporary buffer (truncated)");
399 if (stat(p, &statbuf) == 0) {
401 if ((statbuf.st_mode & S_IWOTH) != 0) {
402 fr_strerror_printf("panic_action file \"%s\" is globally writable", p);
411 /** Prints a simple backtrace (if execinfo is available) and calls panic_action if set.
415 void fr_fault(int sig)
417 char cmd[sizeof(panic_action) + 20];
419 size_t left = sizeof(cmd), ret;
421 char const *p = panic_action;
427 * Makes the backtraces slightly cleaner
429 memset(cmd, 0, sizeof(cmd));
431 FR_FAULT_LOG("CAUGHT SIGNAL: %s", strsignal(sig));
434 * Check for administrator sanity.
436 if (fr_fault_check_permissions() < 0) {
437 FR_FAULT_LOG("Refusing to execute panic action: %s", fr_strerror());
442 * Run the callback if one was registered
444 if (panic_cb && (panic_cb(sig) < 0)) goto finish;
447 * Produce a simple backtrace - They've very basic but at least give us an
448 * idea of the area of the code we hit the issue in.
450 * See below in fr_fault_setup() and
451 * https://sourceware.org/bugzilla/show_bug.cgi?id=16159
452 * for why we only print backtraces in debug builds if we're using GLIBC.
454 #if defined(HAVE_EXECINFO) && (!defined(NDEBUG) || !defined(__GNUC__))
456 size_t frame_count, i;
457 void *stack[MAX_BT_FRAMES];
460 frame_count = backtrace(stack, MAX_BT_FRAMES);
462 FR_FAULT_LOG("Backtrace of last %zu frames:", frame_count);
465 * Only use backtrace_symbols() if we don't have a logging fd.
466 * If the server has experienced memory corruption, there's
467 * a high probability that calling backtrace_symbols() which
468 * mallocs more memory, will fail.
470 if (fr_fault_log_fd < 0) {
471 strings = backtrace_symbols(stack, frame_count);
472 for (i = 0; i < frame_count; i++) {
473 FR_FAULT_LOG("%s", strings[i]);
477 backtrace_symbols_fd(stack, frame_count, fr_fault_log_fd);
482 /* No panic action set... */
483 if (panic_action[0] == '\0') {
484 FR_FAULT_LOG("No panic action set");
488 /* Substitute %p for the current PID (useful for attaching a debugger) */
489 while ((q = strstr(p, "%p"))) {
490 out += ret = snprintf(out, left, "%.*s%d", (int) (q - p), p, (int) getpid());
493 FR_FAULT_LOG("Panic action too long");
499 if (strlen(p) >= left) goto oob;
500 strlcpy(out, p, left);
502 FR_FAULT_LOG("Calling: %s", cmd);
505 bool disable = false;
508 * Here we temporarily enable the dumpable flag so if GBD or LLDB
509 * is called in the panic_action, they can pattach tot he running
512 if (fr_get_dumpable_flag() == 0) {
513 if ((fr_set_dumpable_flag(true) < 0) || !fr_get_dumpable_flag()) {
514 FR_FAULT_LOG("Failed setting dumpable flag, pattach may not work: %s", fr_strerror());
518 FR_FAULT_LOG("Temporarily setting PR_DUMPABLE to 1");
524 * We only want to error out here, if dumpable was originally disabled
525 * and we managed to change the value to enabled, but failed
526 * setting it back to disabled.
529 FR_FAULT_LOG("Resetting PR_DUMPABLE to 0");
530 if (fr_set_dumpable_flag(false) < 0) {
531 FR_FAULT_LOG("Failed reseting dumpable flag to off: %s", fr_strerror());
532 FR_FAULT_LOG("Exiting due to insecure process state");
538 FR_FAULT_LOG("Panic action exited with %i", code);
542 if (sig == SIGUSR1) {
550 /** Work around debuggers which can't backtrace past the signal handler
552 * At least this provides us some information when we get talloc errors.
554 static void _fr_talloc_fault(char const *reason)
556 fr_fault_log("talloc abort: %s\n", reason);
561 /** Wrapper to pass talloc log output to our fr_fault_log function
564 static void _fr_talloc_log(char const *msg)
566 fr_fault_log("%s\n", msg);
569 /** Generate a talloc memory report for a context and print to stderr/stdout
571 * @param ctx to generate a report for, may be NULL in which case the root context is used.
573 int fr_log_talloc_report(TALLOC_CTX *ctx)
579 fd = dup(fr_fault_log_fd);
581 fr_strerror_printf("Couldn't write memory report, failed to dup log fd: %s", fr_syserror(errno));
584 log = fdopen(fd, "w");
587 fr_strerror_printf("Couldn't write memory report, fdopen failed: %s", fr_syserror(errno));
592 fprintf(log, "Current state of talloced memory:\n");
593 talloc_report_full(talloc_null_ctx, log);
595 fprintf(log, "Talloc chunk lineage:\n");
596 fprintf(log, "%p (%s)", ctx, talloc_get_name(ctx));
597 while ((ctx = talloc_parent(ctx))) fprintf(log, " < %p (%s)", ctx, talloc_get_name(ctx));
601 fprintf(log, "Talloc context level %i:\n", i++);
602 talloc_report_full(ctx, log);
603 } while ((ctx = talloc_parent(ctx)) &&
604 (talloc_parent(ctx) != talloc_autofree_ctx) && /* Stop before we hit the autofree ctx */
605 (talloc_parent(ctx) != talloc_null_ctx)); /* Stop before we hit NULL ctx */
613 /** Signal handler to print out a talloc memory report
617 static void _fr_fault_mem_report(int sig)
619 fr_fault_log("CAUGHT SIGNAL: %s\n", strsignal(sig));
621 if (fr_log_talloc_report(NULL) < 0) fr_perror("memreport");
624 static int _fr_disable_null_tracking(UNUSED bool *p)
626 talloc_disable_null_tracking();
630 /** Registers signal handlers to execute panic_action on fatal signal
632 * May be called multiple time to change the panic_action/program.
634 * @param cmd to execute on fault. If present %p will be substituted
635 * for the parent PID before the command is executed, and %e
636 * will be substituted for the currently running program.
637 * @param program Name of program currently executing (argv[0]).
638 * @return 0 on success -1 on failure.
640 int fr_fault_setup(char const *cmd, char const *program)
642 static bool setup = false;
644 char *out = panic_action;
645 size_t left = sizeof(panic_action), ret;
651 /* Substitute %e for the current program */
652 while ((q = strstr(p, "%e"))) {
653 out += ret = snprintf(out, left, "%.*s%s", (int) (q - p), p, program ? program : "");
656 fr_strerror_printf("Panic action too long");
662 if (strlen(p) >= left) goto oob;
663 strlcpy(out, p, left);
665 *panic_action = '\0';
669 * Check for administrator sanity.
671 if (fr_fault_check_permissions() < 0) return -1;
673 /* Unsure what the side effects of changing the signal handler mid execution might be */
676 if (fr_set_signal(SIGSEGV, fr_fault) < 0) return -1;
679 if (fr_set_signal(SIGBUS, fr_fault) < 0) return -1;
682 if (fr_set_signal(SIGABRT, fr_fault) < 0) return -1;
684 * Use this instead of abort so we get a
685 * full backtrace with broken versions of LLDB
687 talloc_set_abort_fn(_fr_talloc_fault);
690 if (fr_set_signal(SIGFPE, fr_fault) < 0) return -1;
694 if (fr_set_signal(SIGUSR1, fr_fault) < 0) return -1;
698 if (fr_set_signal(SIGUSR2, _fr_fault_mem_report) < 0) return -1;
702 * Setup the default logger
704 if (!fr_fault_log) fr_fault_set_log_fn(NULL);
705 talloc_set_log_fn(_fr_talloc_log);
708 * Needed for memory reports
714 tmp = talloc(NULL, bool);
715 talloc_null_ctx = talloc_parent(tmp);
719 * Disable null tracking on exit, else valgrind complains
721 talloc_autofree_ctx = talloc_autofree_context();
722 marker = talloc(talloc_autofree_ctx, bool);
723 talloc_set_destructor(marker, _fr_disable_null_tracking);
726 #if defined(HAVE_MALLOPT) && !defined(NDEBUG)
728 * If were using glibc malloc > 2.4 this scribbles over
729 * uninitialised and freed memory, to make memory issues easier
732 if (!getenv("TALLOC_FREE_FILL")) mallopt(M_PERTURB, 0x42);
733 mallopt(M_CHECK_ACTION, 3);
736 #if defined(HAVE_EXECINFO) && defined(__GNUC__) && !defined(NDEBUG)
738 * We need to pre-load lgcc_s, else we can get into a deadlock
739 * in fr_fault, as backtrace() attempts to dlopen it.
741 * Apparently there's a performance impact of loading lgcc_s,
742 * so only do it if this is a debug build.
744 * See: https://sourceware.org/bugzilla/show_bug.cgi?id=16159
749 backtrace(stack, 10);
758 /** Set a callback to be called before fr_fault()
760 * @param func to execute. If callback returns < 0
761 * fr_fault will exit before running panic_action code.
763 void fr_fault_set_cb(fr_fault_cb_t func)
768 /** Default logger, logs output to stderr
771 static void CC_HINT(format (printf, 1, 2)) _fr_fault_log(char const *msg, ...)
776 vfprintf(stderr, msg, ap);
781 /** Set a file descriptor to log panic_action output to.
783 * @param func to call to output log messages.
785 void fr_fault_set_log_fn(fr_fault_log_t func)
787 fr_fault_log = func ? func : _fr_fault_log;
790 /** Set a file descriptor to log memory reports to.
792 * @param fd to write output to.
794 void fr_fault_set_log_fd(int fd)
796 fr_fault_log_fd = fd;
800 #ifdef WITH_VERIFY_PTR
803 * Verify a VALUE_PAIR
805 inline void fr_verify_vp(char const *file, int line, VALUE_PAIR const *vp)
808 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR pointer was NULL", file, line);
813 (void) talloc_get_type_abort(vp, VALUE_PAIR);
815 if (vp->data.ptr) switch (vp->da->type) {
822 if (!talloc_get_type(vp->data.ptr, uint8_t)) {
823 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" data buffer type should be "
824 "uint8_t but is %s\n", file, line, vp->da->name, talloc_get_name(vp->data.ptr));
825 (void) talloc_get_type_abort(vp->data.ptr, uint8_t);
828 len = talloc_array_length(vp->vp_octets);
829 if (vp->length > len) {
830 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" length %zu is greater than "
831 "uint8_t data buffer length %zu\n", file, line, vp->da->name, vp->length, len);
836 parent = talloc_parent(vp->data.ptr);
838 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" char buffer is not "
839 "parented by VALUE_PAIR %p, instead parented by %p (%s)\n",
840 file, line, vp->da->name,
841 vp, parent, parent ? talloc_get_name(parent) : "NULL");
853 if (!talloc_get_type(vp->data.ptr, char)) {
854 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" data buffer type should be "
855 "char but is %s\n", file, line, vp->da->name, talloc_get_name(vp->data.ptr));
856 (void) talloc_get_type_abort(vp->data.ptr, char);
859 len = (talloc_array_length(vp->vp_strvalue) - 1);
860 if (vp->length > len) {
861 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" length %zu is greater than "
862 "char buffer length %zu\n", file, line, vp->da->name, vp->length, len);
867 if (vp->vp_strvalue[vp->length] != '\0') {
868 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" char buffer not \\0 "
869 "terminated\n", file, line, vp->da->name);
874 parent = talloc_parent(vp->data.ptr);
876 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: VALUE_PAIR \"%s\" uint8_t buffer is not "
877 "parented by VALUE_PAIR %p, instead parented by %p (%s)\n",
878 file, line, vp->da->name,
879 vp, parent, parent ? talloc_get_name(parent) : "NULL");
894 void fr_verify_list(char const *file, int line, TALLOC_CTX *expected, VALUE_PAIR *vps)
900 for (vp = fr_cursor_init(&cursor, &vps);
902 vp = fr_cursor_next(&cursor)) {
905 parent = talloc_parent(vp);
906 if (expected && (parent != expected)) {
907 fprintf(stderr, "CONSISTENCY CHECK FAILED %s[%u]: Expected VALUE_PAIR \"%s\" to be parented "
908 "by %p (%s), instead parented by %p (%s)\n",
909 file, line, vp->da->name,
910 expected, talloc_get_name(expected),
911 parent, parent ? talloc_get_name(parent) : "NULL");
913 fr_log_talloc_report(expected);
914 if (parent) fr_log_talloc_report(parent);