9c03e73af1f22652049450c80d01c2d2330b4ec2
[cyrus-sasl.git] / cmulocal / sasl2.m4
1 # sasl2.m4--sasl2 libraries and includes
2 # Rob Siemborski
3 # $Id: sasl2.m4,v 1.52 2006/05/18 19:25:00 murch Exp $
4
5 # SASL2_CRYPT_CHK
6 # ---------------
7 AC_DEFUN([SASL_GSSAPI_CHK],
8 [AC_REQUIRE([SASL2_CRYPT_CHK])
9 AC_REQUIRE([CMU_SOCKETS])
10 AC_ARG_ENABLE([gssapi],
11               [AC_HELP_STRING([--enable-gssapi=<DIR>],
12                               [enable GSSAPI authentication [yes]])],
13               [gssapi=$enableval],
14               [gssapi=yes])
15 AC_ARG_WITH([gss_impl],
16             [AC_HELP_STRING([--with-gss_impl={heimdal|mit|cybersafe|seam|auto}],
17                             [choose specific GSSAPI implementation [[auto]]])],
18             [gss_impl=$withval],
19             [gss_impl=auto])
20
21 if test "$gssapi" != no; then
22   platform=
23   case "${host}" in
24     *-*-linux*)
25       platform=__linux
26       ;;
27     *-*-hpux*)
28       platform=__hpux
29       ;;
30     *-*-irix*)
31       platform=__irix
32       ;;
33     *-*-solaris2*)
34 # When should we use __sunos?
35       platform=__solaris
36       ;;
37     *-*-aix*)
38 ###_AIX
39       platform=__aix
40       ;;
41     *)
42       AC_WARN([The system type is not recognized. If you believe that CyberSafe GSSAPI works on this platform, please update the configure script])
43       if test "$gss_impl" = "cybersafe"; then
44         AC_ERROR([CyberSafe was forced, cannot continue as platform is not supported])
45       fi
46       ;;
47   esac
48
49   cmu_saved_CPPFLAGS=$CPPFLAGS
50
51   if test -d ${gssapi}; then
52     CPPFLAGS="$CPPFLAGS -I$gssapi/include"
53 # We want to keep -I in our CPPFLAGS, but only if we succeed
54     cmu_saved_CPPFLAGS=$CPPFLAGS
55 ### I am not sure how useful is this (and whether this is required at all
56 ### especially when we have to provide two -L flags for new CyberSafe
57     LDFLAGS="$LDFLAGS -L$gssapi/lib"
58
59     if test -n "$platform"; then
60       if test "$gss_impl" = "auto" -o "$gss_impl" = "cybersafe"; then
61         CPPFLAGS="$CPPFLAGS -D$platform"
62         if test -d "${gssapi}/appsec-sdk/include"; then
63           CPPFLAGS="$CPPFLAGS -I${gssapi}/appsec-sdk/include"
64         fi
65       fi
66     fi
67   fi
68   AC_CHECK_HEADER([gssapi.h],
69                   [AC_DEFINE(HAVE_GSSAPI_H,,
70                              [Define if you have the gssapi.h header file])],
71                   [AC_CHECK_HEADER([gssapi/gssapi.h],,
72                                    [AC_WARN([Disabling GSSAPI - no include files found]); gssapi=no])])
73   AC_CHECK_HEADERS(gssapi/gssapi_ext.h)
74   CPPFLAGS=$cmu_saved_CPPFLAGS
75
76 fi
77
78 if test "$gssapi" != no; then
79   # We need to find out which gssapi implementation we are
80   # using. Supported alternatives are: MIT Kerberos 5,
81   # Heimdal Kerberos 5 (http://www.pdc.kth.se/heimdal),
82   # CyberSafe Kerberos 5 (http://www.cybersafe.com/)
83   # and Sun SEAM (http://wwws.sun.com/software/security/kerberos/)
84   #
85   # The choice is reflected in GSSAPIBASE_LIBS
86
87   AC_CHECK_LIB(resolv,res_search)
88   if test -d ${gssapi}; then
89      gssapi_dir="${gssapi}/lib"
90      GSSAPIBASE_LIBS="-L$gssapi_dir"
91      GSSAPIBASE_STATIC_LIBS="-L$gssapi_dir"
92   else
93      # FIXME: This is only used for building cyrus, and then only as
94      # a real hack.  it needs to be fixed.
95      gssapi_dir="/usr/local/lib"
96   fi
97
98   # Check a full link against the Heimdal libraries.
99   # If this fails, check a full link against the MIT libraries.
100   # If this fails, check a full link against the CyberSafe libraries.
101   # If this fails, check a full link against the Solaris 8 and up libgss.
102
103   if test "$gss_impl" = "auto" -o "$gss_impl" = "heimdal"; then
104     gss_failed=0
105     AC_CHECK_LIB(gssapi,gss_unwrap,gss_impl="heimdal",gss_failed=1,
106                  ${GSSAPIBASE_LIBS} -lgssapi -lkrb5 -lasn1 -lroken ${LIB_CRYPT} ${LIB_DES} -lcom_err ${LIB_SOCKET})
107     if test "$gss_impl" != "auto" -a "$gss_failed" = "1"; then
108       gss_impl="failed"
109     fi
110   fi
111
112   if test "$gss_impl" = "auto" -o "$gss_impl" = "mit"; then
113     # check for libkrb5support first
114     AC_CHECK_LIB(krb5support,krb5int_getspecific,K5SUP=-lkrb5support K5SUPSTATIC=$gssapi_dir/libkrb5support.a,,${LIB_SOCKET})
115
116     gss_failed=0
117     AC_CHECK_LIB(gssapi_krb5,gss_unwrap,gss_impl="mit",gss_failed=1,
118                  ${GSSAPIBASE_LIBS} -lgssapi_krb5 -lkrb5 -lk5crypto -lcom_err ${K5SUP} ${LIB_SOCKET})
119     if test "$gss_impl" != "auto" -a "$gss_failed" = "1"; then
120       gss_impl="failed"
121     fi
122   fi
123
124   # For Cybersafe one has to set a platform define in order to make compilation work
125   if test "$gss_impl" = "auto" -o "$gss_impl" = "cybersafe"; then
126
127     cmu_saved_CPPFLAGS=$CPPFLAGS
128     cmu_saved_GSSAPIBASE_LIBS=$GSSAPIBASE_LIBS
129 # FIXME - Note that the libraries are in .../lib64 for 64bit kernels
130     if test -d "${gssapi}/appsec-rt/lib"; then
131       GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -L${gssapi}/appsec-rt/lib"
132     fi
133     CPPFLAGS="$CPPFLAGS -D$platform"
134     if test -d "${gssapi}/appsec-sdk/include"; then
135       CPPFLAGS="$CPPFLAGS -I${gssapi}/appsec-sdk/include"
136     fi
137
138     gss_failed=0
139
140 # Check for CyberSafe with two libraries first, than fall back to a single 
141 # library (older CyberSafe)
142
143     unset ac_cv_lib_gss_csf_gss_acq_user
144     AC_CHECK_LIB(gss,csf_gss_acq_user,gss_impl="cybersafe03",
145                  [unset ac_cv_lib_gss_csf_gss_acq_user;
146                   AC_CHECK_LIB(gss,csf_gss_acq_user,gss_impl="cybersafe",
147                                gss_failed=1,$GSSAPIBASE_LIBS -lgss)],
148                  [${GSSAPIBASE_LIBS} -lgss -lcstbk5])
149
150     if test "$gss_failed" = "1"; then
151 # Restore variables
152       GSSAPIBASE_LIBS=$cmu_saved_GSSAPIBASE_LIBS
153       CPPFLAGS=$cmu_saved_CPPFLAGS
154
155       if test "$gss_impl" != "auto"; then
156         gss_impl="failed"
157       fi
158     fi
159   fi
160
161   if test "$gss_impl" = "auto" -o "$gss_impl" = "seam"; then
162     gss_failed=0
163     AC_CHECK_LIB(gss,gss_unwrap,gss_impl="seam",gss_failed=1,-lgss)
164     if test "$gss_impl" != "auto" -a "$gss_failed" = "1"; then
165       gss_impl="failed"
166     fi
167   fi
168
169   if test "$gss_impl" = "mit"; then
170     GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -lgssapi_krb5 -lkrb5 -lk5crypto -lcom_err ${K5SUP}"
171     GSSAPIBASE_STATIC_LIBS="$GSSAPIBASE_LIBS $gssapi_dir/libgssapi_krb5.a $gssapi_dir/libkrb5.a $gssapi_dir/libk5crypto.a $gssapi_dir/libcom_err.a ${K5SUPSTATIC}"
172   elif test "$gss_impl" = "heimdal"; then
173     CPPFLAGS="$CPPFLAGS -DKRB5_HEIMDAL"
174     GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -lgssapi -lkrb5 -lasn1 -lroken ${LIB_CRYPT} ${LIB_DES} -lcom_err"
175     GSSAPIBASE_STATIC_LIBS="$GSSAPIBASE_STATIC_LIBS $gssapi_dir/libgssapi.a $gssapi_dir/libkrb5.a $gssapi_dir/libasn1.a $gssapi_dir/libroken.a $gssapi_dir/libcom_err.a ${LIB_CRYPT}"
176   elif test "$gss_impl" = "cybersafe03"; then
177 # Version of CyberSafe with two libraries
178     CPPFLAGS="$CPPFLAGS -D$platform -I${gssapi}/appsec-sdk/include"
179     GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -lgss -lcstbk5"
180     # there is no static libgss for CyberSafe
181     GSSAPIBASE_STATIC_LIBS=none
182   elif test "$gss_impl" = "cybersafe"; then
183     CPPFLAGS="$CPPFLAGS -D$platform -I${gssapi}/appsec-sdk/include"
184     GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -lgss"
185     # there is no static libgss for CyberSafe
186     GSSAPIBASE_STATIC_LIBS=none
187   elif test "$gss_impl" = "seam"; then
188     GSSAPIBASE_LIBS=-lgss
189     # there is no static libgss on Solaris 8 and up
190     GSSAPIBASE_STATIC_LIBS=none
191   elif test "$gss_impl" = "failed"; then
192     gssapi="no"
193     GSSAPIBASE_LIBS=
194     GSSAPIBASE_STATIC_LIBS=
195     AC_WARN([Disabling GSSAPI - specified library not found])
196   else
197     gssapi="no"
198     GSSAPIBASE_LIBS=
199     GSSAPIBASE_STATIC_LIBS=
200     AC_WARN([Disabling GSSAPI - no library])
201   fi
202 fi
203
204 #
205 # Cybersafe defines both GSS_C_NT_HOSTBASED_SERVICE and GSS_C_NT_USER_NAME
206 # in gssapi\rfckrb5.h
207 #
208 if test "$gssapi" != "no"; then
209   if test "$gss_impl" = "cybersafe" -o "$gss_impl" = "cybersafe03"; then
210     AC_EGREP_CPP(hostbased_service_gss_nt_yes,
211                  [#include <gssapi/gssapi.h>
212                   #ifdef GSS_C_NT_HOSTBASED_SERVICE
213                     hostbased_service_gss_nt_yes
214                   #endif],
215                  [AC_DEFINE(HAVE_GSS_C_NT_HOSTBASED_SERVICE,,
216                             [Define if your GSSAPI implimentation defines GSS_C_NT_HOSTBASED_SERVICE])],
217                  [AC_WARN([Cybersafe define not found])])
218
219   elif test "$ac_cv_header_gssapi_h" = "yes"; then
220     AC_EGREP_HEADER(GSS_C_NT_HOSTBASED_SERVICE, gssapi.h,
221                     [AC_DEFINE(HAVE_GSS_C_NT_HOSTBASED_SERVICE,,
222                                [Define if your GSSAPI implimentation defines GSS_C_NT_HOSTBASED_SERVICE])])
223   elif test "$ac_cv_header_gssapi_gssapi_h"; then
224     AC_EGREP_HEADER(GSS_C_NT_HOSTBASED_SERVICE, gssapi/gssapi.h,
225                     [AC_DEFINE(HAVE_GSS_C_NT_HOSTBASED_SERVICE,,
226                                [Define if your GSSAPI implimentation defines GSS_C_NT_HOSTBASED_SERVICE])])
227   fi
228
229   if test "$gss_impl" = "cybersafe" -o "$gss_impl" = "cybersafe03"; then
230     AC_EGREP_CPP(user_name_yes_gss_nt,
231                  [#include <gssapi/gssapi.h>
232                   #ifdef GSS_C_NT_USER_NAME
233                    user_name_yes_gss_nt
234                   #endif],
235                  [AC_DEFINE(HAVE_GSS_C_NT_USER_NAME,,
236                             [Define if your GSSAPI implimentation defines GSS_C_NT_USER_NAME])],
237                  [AC_WARN([Cybersafe define not found])])
238   elif test "$ac_cv_header_gssapi_h" = "yes"; then
239     AC_EGREP_HEADER(GSS_C_NT_USER_NAME, gssapi.h,
240                     [AC_DEFINE(HAVE_GSS_C_NT_USER_NAME,,
241                                [Define if your GSSAPI implimentation defines GSS_C_NT_USER_NAME])])
242     AC_EGREP_HEADER(gss_inquire_attrs_for_mech, gssapi.h, rfc5587=yes)
243   elif test "$ac_cv_header_gssapi_gssapi_h"; then
244     AC_EGREP_HEADER(GSS_C_NT_USER_NAME, gssapi/gssapi.h,
245                     [AC_DEFINE(HAVE_GSS_C_NT_USER_NAME,,
246                                [Define if your GSSAPI implimentation defines GSS_C_NT_USER_NAME])])
247     AC_EGREP_HEADER(gss_inquire_attrs_for_mech, gssapi/gssapi.h, rfc5587=yes)
248   fi
249 fi
250
251 GSSAPI_LIBS=""
252 AC_MSG_CHECKING([GSSAPI])
253 if test "$gssapi" != no; then
254   AC_MSG_RESULT([with implementation ${gss_impl}])
255   AC_CHECK_LIB(resolv,res_search,GSSAPIBASE_LIBS="$GSSAPIBASE_LIBS -lresolv")
256   SASL_MECHS="$SASL_MECHS libgssapiv2.la"
257   SASL_STATIC_OBJS="$SASL_STATIC_OBJS gssapi.o"
258   SASL_STATIC_SRCS="$SASL_STATIC_SRCS ../plugins/gssapi.c"
259   if test "$rfc5587" = "yes"; then
260     SASL_MECHS="$SASL_MECHS libgs2.la"
261     SASL_STATIC_OBJS="$SASL_STATIC_OBJS gs2.o"
262     SASL_STATIC_SRCS="$SASL_STATIC_SRCS ../plugins/gs2.c"
263   fi
264   cmu_save_LIBS="$LIBS"
265   LIBS="$LIBS $GSSAPIBASE_LIBS"
266   AC_CHECK_FUNCS(gsskrb5_register_acceptor_identity)
267   LIBS="$cmu_save_LIBS"
268 else
269   AC_MSG_RESULT([disabled])
270 fi
271 AC_SUBST(GSSAPI_LIBS)
272 AC_SUBST(GSSAPIBASE_LIBS)
273 ])# SASL_GSSAPI_CHK
274
275
276 # SASL_SET_GSSAPI_LIBS
277 # --------------------
278 AC_DEFUN([SASL_SET_GSSAPI_LIBS],
279 [SASL_GSSAPI_LIBS_SET="yes"
280 ])
281
282
283 # CMU_SASL2
284 # ---------
285 # What we want to do here is setup LIB_SASL with what one would
286 # generally want to have (e.g. if static is requested, make it that,
287 # otherwise make it dynamic.
288 #
289 # We also want to create LIB_DYN_SASL and DYNSASLFLAGS.
290 #
291 # Also sets using_static_sasl to "no" "static" or "staticonly"
292 #
293 AC_DEFUN([CMU_SASL2],
294 [AC_REQUIRE([SASL_GSSAPI_CHK])
295
296 AC_ARG_WITH(sasl,
297             [AC_HELP_STRING([--with-sasl=DIR],[Compile with libsasl2 in <DIR>])],
298             with_sasl="$withval",
299             with_sasl="yes")
300
301 AC_ARG_WITH(staticsasl,
302             [AC_HELP_STRING([--with-staticsasl=DIR],
303                             [Compile with staticly linked libsasl2 in <DIR>])],
304             [with_staticsasl="$withval";
305              if test $with_staticsasl != "no"; then
306                using_static_sasl="static"
307              fi],
308             [with_staticsasl="no"; using_static_sasl="no"])
309
310 SASLFLAGS=""
311 LIB_SASL=""
312
313 cmu_saved_CPPFLAGS=$CPPFLAGS
314 cmu_saved_LDFLAGS=$LDFLAGS
315 cmu_saved_LIBS=$LIBS
316
317 if test ${with_staticsasl} != "no"; then
318   if test -d ${with_staticsasl}; then
319     if test -d ${with_staticsasl}/lib64 ; then
320       ac_cv_sasl_where_lib=${with_staticsasl}/lib64
321     else
322       ac_cv_sasl_where_lib=${with_staticsasl}/lib
323     fi
324     ac_cv_sasl_where_lib=${with_staticsasl}/lib
325     ac_cv_sasl_where_inc=${with_staticsasl}/include
326
327     SASLFLAGS="-I$ac_cv_sasl_where_inc"
328     LIB_SASL="-L$ac_cv_sasl_where_lib"
329     CPPFLAGS="${cmu_saved_CPPFLAGS} -I${ac_cv_sasl_where_inc}"
330     LDFLAGS="${cmu_saved_LDFLAGS} -L${ac_cv_sasl_where_lib}"
331   else
332     with_staticsasl="/usr"
333   fi
334
335   AC_CHECK_HEADER(sasl/sasl.h,
336                   [AC_CHECK_HEADER(sasl/saslutil.h,
337                                    [for i42 in lib64 lib; do
338                                       if test -r ${with_staticsasl}/$i42/libsasl2.a; then
339                                         ac_cv_found_sasl=yes
340                                         AC_MSG_CHECKING([for static libsasl])
341                                         LIB_SASL="$LIB_SASL ${with_staticsasl}/$i42/libsasl2.a"
342                                       fi
343                                     done
344                                     if test ! "$ac_cv_found_sasl" = "yes"; then
345                                       AC_MSG_CHECKING([for static libsasl])
346                                       AC_ERROR([Could not find ${with_staticsasl}/lib*/libsasl2.a])
347                                     fi])])
348
349   AC_MSG_RESULT([found])
350
351   if test "x$SASL_GSSAPI_LIBS_SET" = "x"; then
352     LIB_SASL="$LIB_SASL $GSSAPIBASE_STATIC_LIBS"
353   else
354     SASL_GSSAPI_LIBS_SET=""
355     cmu_saved_LIBS="$GSSAPIBASE_STATIC_LIBS $cmu_saved_LIBS" 
356   fi
357 fi
358
359 if test -d ${with_sasl}; then
360   ac_cv_sasl_where_lib=${with_sasl}/lib
361   ac_cv_sasl_where_inc=${with_sasl}/include
362
363   DYNSASLFLAGS="-I$ac_cv_sasl_where_inc"
364   if test "$ac_cv_sasl_where_lib" != ""; then
365     CMU_ADD_LIBPATH_TO($ac_cv_sasl_where_lib, LIB_DYN_SASL)
366   fi
367   LIB_DYN_SASL="$LIB_DYN_SASL -lsasl2"
368   CPPFLAGS="${cmu_saved_CPPFLAGS} -I${ac_cv_sasl_where_inc}"
369   LDFLAGS="${cmu_saved_LDFLAGS} -L${ac_cv_sasl_where_lib}"
370 fi
371
372 # be sure to check for a SASLv2 specific function
373 AC_CHECK_HEADER(sasl/sasl.h,
374                 [AC_CHECK_HEADER(sasl/saslutil.h,
375                                  [AC_CHECK_LIB(sasl2, prop_get, 
376                                                ac_cv_found_sasl=yes,
377                                                ac_cv_found_sasl=no)],
378                                  ac_cv_found_sasl=no)],
379                 ac_cv_found_sasl=no)
380
381 if test "$ac_cv_found_sasl" = "yes"; then
382   if test "$ac_cv_sasl_where_lib" != ""; then
383     CMU_ADD_LIBPATH_TO($ac_cv_sasl_where_lib, DYNLIB_SASL)
384   fi
385   DYNLIB_SASL="$DYNLIB_SASL -lsasl2"
386   if test "$using_static_sasl" != "static"; then
387     LIB_SASL=$DYNLIB_SASL
388     SASLFLAGS=$DYNSASLFLAGS
389   fi
390 else
391   DYNLIB_SASL=""
392   DYNSASLFLAGS=""
393   using_static_sasl="staticonly"
394 fi
395
396 if test "x$SASL_GSSAPI_LIBS_SET" != "x"; then
397   SASL_GSSAPI_LIBS_SET=""
398   cmu_saved_LIBS="$GSSAPIBASE_LIBS $cmu_saved_LIBS" 
399 fi
400
401 LIBS="$cmu_saved_LIBS"
402 LDFLAGS="$cmu_saved_LDFLAGS"
403 CPPFLAGS="$cmu_saved_CPPFLAGS"
404
405 AC_SUBST(LIB_DYN_SASL)
406 AC_SUBST(DYNSASLFLAGS)
407 AC_SUBST(LIB_SASL)
408 AC_SUBST(SASLFLAGS)
409 ])# CMU_SASL2
410
411
412 # CMU_SASL2_REQUIRED
413 # ------------------
414 AC_DEFUN([CMU_SASL2_REQUIRED],
415 [AC_REQUIRE([CMU_SASL2])
416 if test "$ac_cv_found_sasl" != "yes"; then
417   AC_ERROR([Cannot continue without libsasl2.
418 Get it from ftp://ftp.andrew.cmu.edu/pub/cyrus-mail/.])
419 fi])
420
421
422 # CMU_SASL2_REQUIRE_VER
423 # ---------------------
424 AC_DEFUN([CMU_SASL2_REQUIRE_VER],
425 [AC_REQUIRE([CMU_SASL2_REQUIRED])
426
427 cmu_saved_CPPFLAGS=$CPPFLAGS
428 CPPFLAGS="$CPPFLAGS $SASLFLAGS"
429
430 AC_TRY_CPP([
431 #include <sasl/sasl.h>
432
433 #ifndef SASL_VERSION_MAJOR
434 #error SASL_VERSION_MAJOR not defined
435 #endif
436 #ifndef SASL_VERSION_MINOR
437 #error SASL_VERSION_MINOR not defined
438 #endif
439 #ifndef SASL_VERSION_STEP
440 #error SASL_VERSION_STEP not defined
441 #endif
442
443 #if SASL_VERSION_MAJOR < $1 || SASL_VERSION_MINOR < $2 || SASL_VERSION_STEP < $3
444 #error SASL version is less than $1.$2.$3
445 #endif
446 ],,
447            [AC_ERROR([Incorrect SASL headers found.  This package requires SASL $1.$2.$3 or newer.])])
448
449 CPPFLAGS=$cmu_saved_CPPFLAGS
450 ])# CMU_SASL2_REQUIRE_VER
451
452
453 # CMU_SASL2_CHECKAPOP_REQUIRED
454 # ----------------------------
455 AC_DEFUN([CMU_SASL2_CHECKAPOP_REQUIRED],
456 [AC_REQUIRE([CMU_SASL2_REQUIRED])
457
458 cmu_saved_LDFLAGS=$LDFLAGS
459
460 LDFLAGS="$LDFLAGS $LIB_SASL"
461
462 AC_CHECK_LIB(sasl2, sasl_checkapop,
463              [AC_DEFINE(HAVE_APOP,[],[Does SASL support APOP?])],
464              [AC_MSG_ERROR([libsasl2 without working sasl_checkapop.  Cannot continue.])])
465
466 LDFLAGS=$cmu_saved_LDFLAGS
467 ])# CMU_SASL2_CHECKAPOP_REQUIRED
468
469
470 # SASL2_CRYPT_CHK
471 # ---------------
472 AC_DEFUN([SASL2_CRYPT_CHK],
473 [AC_CHECK_FUNC(crypt, cmu_have_crypt=yes,
474                [AC_CHECK_LIB(crypt, crypt,
475                              LIB_CRYPT="-lcrypt"; cmu_have_crypt=yes,
476                              cmu_have_crypt=no)])
477 AC_SUBST(LIB_CRYPT)
478 ])# SASL2_CRYPT_CHK