Import the new apropos/whatis. trunk
authorjoerg <joerg@NetBSD.org>
Tue, 07 Feb 2012 19:13:24 +0000
branchtrunk
changeset 209019 bae77d7bf562
parent 209018 413896988988
child 209020 ce0d26e4523c
Import the new apropos/whatis. This code has been developed by Abhinav Upadhyay as part of Google's Summer of Code 2011. It uses libmandoc to parse man pages and builds a Full Text Index in a SQLite database. The combination of indexing the full manual page, filtering out stop words and ranking individual matches based on the section gives a much improved user experience. The old makewhatis and friends are kept under MKMAKEMANDB=no for now.
distrib/sets/lists/base/mi
distrib/sets/lists/comp/mi
distrib/sets/lists/etc/mi
distrib/sets/lists/man/mi
distrib/sets/lists/text/mi
distrib/sets/lists/xbase/mi
distrib/sets/sets.subr
etc/daily
etc/defaults/daily.conf
etc/defaults/rc.conf
etc/defaults/weekly.conf
etc/rc.d/Makefile
etc/rc.d/makemandb
etc/weekly
lib/Makefile
libexec/Makefile
share/man/Makefile
share/man/man0/Makefile
share/mk/bsd.own.mk
usr.bin/Makefile
usr.sbin/Makefile
usr.sbin/makemandb/DBSCHEMA
usr.sbin/makemandb/Makefile
usr.sbin/makemandb/apropos-utils.3
usr.sbin/makemandb/apropos-utils.c
usr.sbin/makemandb/apropos-utils.h
usr.sbin/makemandb/apropos.1
usr.sbin/makemandb/apropos.c
usr.sbin/makemandb/close_db.3
usr.sbin/makemandb/init_db.3
usr.sbin/makemandb/makemandb.8
usr.sbin/makemandb/makemandb.c
usr.sbin/makemandb/run_query.3
usr.sbin/makemandb/run_query_html.3
usr.sbin/makemandb/run_query_pager.3
usr.sbin/makemandb/stopwords.txt
usr.sbin/makemandb/whatis.1
usr.sbin/makemandb/whatis.c
--- a/distrib/sets/lists/base/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/base/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.980 2012/02/05 14:19:04 dholland Exp $
+# $NetBSD: mi,v 1.981 2012/02/07 19:13:25 joerg Exp $
 #
 # Note:	Don't delete entries from here - mark them as "obsolete" instead,
 #	unless otherwise stated below.
@@ -1023,7 +1023,7 @@
 ./usr/libexec/fingerd				base-netutil-bin
 ./usr/libexec/ftp-proxy				base-obsolete		obsolete
 ./usr/libexec/ftpd				base-netutil-bin
-./usr/libexec/getNAME				base-man-bin
+./usr/libexec/getNAME				base-man-bin		!makemandb
 ./usr/libexec/getty				base-sysutil-bin
 ./usr/libexec/hprop				base-obsolete		obsolete
 ./usr/libexec/hpropd				base-krb5-bin		kerberos
@@ -1044,7 +1044,7 @@
 ./usr/libexec/lpr/pjlfilter			base-lpr-bin
 ./usr/libexec/mail.local			base-mail-bin
 ./usr/libexec/makekey				base-crypto-bin
-./usr/libexec/makewhatis			base-man-bin
+./usr/libexec/makewhatis			base-man-bin		!makemandb
 ./usr/libexec/named-xfer			base-obsolete		obsolete
 ./usr/libexec/ntalkd				base-netutil-bin
 ./usr/libexec/pfspamd				base-obsolete		obsolete
@@ -3614,7 +3614,7 @@
 ./usr/share/man/html9/vax			base-man-share
 ./usr/share/man/html9/x68k			base-man-share
 ./usr/share/man/html9/x86			base-man-share
-./usr/share/man/makewhatis.sed			base-man-share		share
+./usr/share/man/makewhatis.sed			base-man-share		!makemandb,share
 ./usr/share/man/man1				base-man-share
 ./usr/share/man/man1/atari			base-man-share
 ./usr/share/man/man1/hp300			base-man-share
@@ -3742,7 +3742,7 @@
 ./usr/share/man/man9/vax			base-man-share
 ./usr/share/man/man9/x68k			base-man-share
 ./usr/share/man/man9/x86			base-man-share
-./usr/share/man/whatis.db			base-man-share		man
+./usr/share/man/whatis.db			base-man-share		man,!makemandb
 ./usr/share/me					base-groff-share
 ./usr/share/misc				base-terminfo-share
 ./usr/share/misc/acronyms			base-reference-share	share
--- a/distrib/sets/lists/comp/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/comp/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: mi,v 1.1730 2012/02/05 14:19:05 dholland Exp $
+#	$NetBSD: mi,v 1.1731 2012/02/07 19:13:25 joerg Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 #
@@ -3971,7 +3971,7 @@
 ./usr/libdata/debug/usr/libexec/fingerd.debug	comp-netutil-debug	debug
 ./usr/libdata/debug/usr/libexec/ftp-proxy.debug	comp-obsolete		obsolete
 ./usr/libdata/debug/usr/libexec/ftpd.debug	comp-netutil-debug	debug
-./usr/libdata/debug/usr/libexec/getNAME.debug	comp-man-debug		debug
+./usr/libdata/debug/usr/libexec/getNAME.debug	comp-man-debug		!makemandb,debug
 ./usr/libdata/debug/usr/libexec/getty.debug	comp-sysutil-debug	debug
 ./usr/libdata/debug/usr/libexec/hpropd.debug	comp-krb5-debug		kerberos,debug
 ./usr/libdata/debug/usr/libexec/httpd.debug	comp-netutil-debug	debug
@@ -3988,7 +3988,7 @@
 ./usr/libdata/debug/usr/libexec/lpr/lpf.debug	comp-lpr-debug		debug
 ./usr/libdata/debug/usr/libexec/mail.local.debug	comp-mail-debug		debug
 ./usr/libdata/debug/usr/libexec/makekey.debug	comp-crypto-debug	debug
-./usr/libdata/debug/usr/libexec/makewhatis.debug	comp-man-debug		debug
+./usr/libdata/debug/usr/libexec/makewhatis.debug	comp-man-debug		!makemandb,debug
 ./usr/libdata/debug/usr/libexec/ntalkd.debug	comp-netutil-debug	debug
 ./usr/libdata/debug/usr/libexec/pcpp.debug	comp-c-debug		pcc,debug
 ./usr/libdata/debug/usr/libexec/postfix/anvil.debug	comp-postfix-debug	postfix,debug
@@ -4148,6 +4148,7 @@
 ./usr/libdata/debug/usr/sbin/lpd.debug		comp-lpr-debug		debug
 ./usr/libdata/debug/usr/sbin/lptest.debug	comp-lpr-debug		debug
 ./usr/libdata/debug/usr/sbin/mailwrapper.debug	comp-mailwrapper-debug	debug
+./usr/libdata/debug/usr/sbin//makemandb.debug	comp-makemandb-debug	makemandb,debug
 ./usr/libdata/debug/usr/sbin/makedbm.debug	comp-nis-debug		yp,debug
 ./usr/libdata/debug/usr/sbin/makefs.debug	comp-sysutil-debug	debug
 ./usr/libdata/debug/usr/sbin/map-mbone.debug	comp-netutil-debug	debug
--- a/distrib/sets/lists/etc/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/etc/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.227 2011/12/17 13:37:49 tsutsui Exp $
+# $NetBSD: mi,v 1.228 2012/02/07 19:13:28 joerg Exp $
 #
 # Note: end-user configuration files that are moved to another location
 #	should not be marked "obsolete"; they should just be removed from
@@ -223,6 +223,7 @@
 ./etc/rc.d/local				etc-sys-rc
 ./etc/rc.d/lpd					etc-lpr-rc
 ./etc/rc.d/lvm					etc-sys-rc
+./etc/rc.d/makemandb				etc-man-rc
 ./etc/rc.d/mdnsd				etc-mdns-rc
 ./etc/rc.d/mixerctl				etc-audio-rc
 ./etc/rc.d/mopd					etc-bootserver-rc
--- a/distrib/sets/lists/man/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/man/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.1375 2012/02/03 22:18:05 joerg Exp $
+# $NetBSD: mi,v 1.1376 2012/02/07 19:13:29 joerg Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 #
@@ -2313,7 +2313,7 @@
 ./usr/share/man/cat8/fusermount.0		man-refuse-catman	.cat
 ./usr/share/man/cat8/fwctl.0			man-sysutil-catman	.cat
 ./usr/share/man/cat8/genassym.sh.0		man-obsolete		obsolete
-./usr/share/man/cat8/getNAME.0			man-man-catman		.cat
+./usr/share/man/cat8/getNAME.0			man-man-catman		!makemandb, .cat
 ./usr/share/man/cat8/getencstat.0		man-sysutil-catman	.cat
 ./usr/share/man/cat8/gettable.0			man-obsolete		obsolete
 ./usr/share/man/cat8/getty.0			man-sysutil-catman	.cat
@@ -2477,8 +2477,9 @@
 ./usr/share/man/cat8/makedev.local.0		man-sysutil-catman	.cat
 ./usr/share/man/cat8/makefs.0			man-sysutil-catman	.cat
 ./usr/share/man/cat8/makekey.0			man-crypto-catman	.cat
+./usr/share/man/cat8/makemandb.0		man-man-catman		makemandb,.cat
 ./usr/share/man/cat8/makemap.0			man-obsolete		obsolete
-./usr/share/man/cat8/makewhatis.0		man-man-catman		.cat
+./usr/share/man/cat8/makewhatis.0		man-man-catman		!makemandb,.cat
 ./usr/share/man/cat8/map-mbone.0		man-netutil-catman	.cat
 ./usr/share/man/cat8/master.0			man-postfix-catman	postfix,.cat
 ./usr/share/man/cat8/mbr.0			man-obsolete		obsolete
@@ -5015,7 +5016,7 @@
 ./usr/share/man/html8/ftpd.html			man-netutil-htmlman	html
 ./usr/share/man/html8/fusermount.html		man-refuse-htmlman	html
 ./usr/share/man/html8/fwctl.html		man-sysutil-htmlman	html
-./usr/share/man/html8/getNAME.html		man-man-htmlman		html
+./usr/share/man/html8/getNAME.html		man-man-htmlman		!makemandb,html
 ./usr/share/man/html8/getencstat.html		man-sysutil-htmlman	html
 ./usr/share/man/html8/getty.html		man-sysutil-htmlman	html
 ./usr/share/man/html8/gpioctl.html		man-sysutil-htmlman	html
@@ -5141,7 +5142,8 @@
 ./usr/share/man/html8/makedev.local.html	man-sysutil-htmlman	html
 ./usr/share/man/html8/makefs.html		man-sysutil-htmlman	html
 ./usr/share/man/html8/makekey.html		man-crypto-htmlman	html
-./usr/share/man/html8/makewhatis.html		man-man-htmlman		html
+./usr/share/man/html8/makemandb.html		man-man-htmlman		makemandb,html
+./usr/share/man/html8/makewhatis.html		man-man-htmlman		!makemandb,html
 ./usr/share/man/html8/map-mbone.html		man-netutil-htmlman	html
 ./usr/share/man/html8/master.html		man-postfix-htmlman	postfix,html
 ./usr/share/man/html8/mbrlabel.html		man-sysutil-htmlman	html
@@ -7811,7 +7813,7 @@
 ./usr/share/man/man8/fusermount.8		man-refuse-man		.man
 ./usr/share/man/man8/fwctl.8			man-sysutil-man		.man
 ./usr/share/man/man8/genassym.sh.8		man-obsolete		obsolete
-./usr/share/man/man8/getNAME.8			man-man-man		.man
+./usr/share/man/man8/getNAME.8			man-man-man		!makemandb,.man
 ./usr/share/man/man8/getencstat.8		man-sysutil-man		.man
 ./usr/share/man/man8/gettable.8			man-obsolete		obsolete
 ./usr/share/man/man8/getty.8			man-sysutil-man		.man
@@ -7976,8 +7978,9 @@
 ./usr/share/man/man8/makedev.local.8		man-sysutil-man		.man
 ./usr/share/man/man8/makefs.8			man-sysutil-man		.man
 ./usr/share/man/man8/makekey.8			man-crypto-man		.man
+./usr/share/man/man8/makemandb.8		man-man-man		makemandb,.man
 ./usr/share/man/man8/makemap.8			man-obsolete		obsolete
-./usr/share/man/man8/makewhatis.8		man-man-man		.man
+./usr/share/man/man8/makewhatis.8		man-man-man		!makemandb,.man
 ./usr/share/man/man8/map-mbone.8		man-netutil-man		.man
 ./usr/share/man/man8/master.8			man-postfix-man		postfix,.man
 ./usr/share/man/man8/mbr.8			man-obsolete		obsolete
--- a/distrib/sets/lists/text/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/text/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.41 2011/10/11 19:20:13 joerg Exp $
+# $NetBSD: mi,v 1.42 2012/02/07 19:13:29 joerg Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 #
@@ -40,6 +40,7 @@
 ./usr/bin/troff					text-groff-bin		groff
 ./usr/bin/vgrind				text-groff-bin
 ./usr/libexec/vfontedpr				text-groff-bin
+./usr/sbin/makemandb				base-man-bin		makemandb
 ./usr/share/doc/groff				text-groff-doc
 ./usr/share/doc/groff/mom			text-groff-doc
 ./usr/share/doc/groff/mom/README.txt		text-groff-doc		share,groff
--- a/distrib/sets/lists/xbase/mi	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/lists/xbase/mi	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: mi,v 1.111 2011/10/22 22:23:50 jmcneill Exp $
+# $NetBSD: mi,v 1.112 2012/02/07 19:13:30 joerg Exp $
 #
 # Note: don't delete entries from here - mark them as "obsolete" instead.
 #
@@ -1340,7 +1340,7 @@
 ./usr/X11R6/man/man7/XProjectTeam.7			xbase-obsolete	obsolete
 ./usr/X11R6/man/man7/XStandards.7			-unknown-	.man,x11
 ./usr/X11R6/man/man7/Xsecurity.7			-unknown-	.man,x11
-./usr/X11R6/man/whatis.db				-unknown-	man,x11
+./usr/X11R6/man/whatis.db				-unknown-	man,x11,!makemandb
 ./usr/X11R6/share					base-x11-root	x11
 ./usr/X11R6/share/examples				base-x11-root	x11
 ./usr/X11R6/share/examples/ssh-askpass			base-x11-root	x11
@@ -2888,7 +2888,7 @@
 ./usr/X11R7/man/man7/XOrgFoundation.7			-unknown-	.man,xorg
 ./usr/X11R7/man/man7/XProjectTeam.7			-unknown-	.man,xorg
 ./usr/X11R7/man/man7/Xsecurity.7			-unknown-	.man,xorg
-./usr/X11R7/man/whatis.db				-unknown-	man,xorg
+./usr/X11R7/man/whatis.db				-unknown-	man,xorg,!makemandb
 ./usr/X11R7/share					base-x11-root	xorg
 ./usr/X11R7/share/aclocal				base-x11-root	xorg
 ./usr/X11R7/share/examples				base-x11-root	xorg
--- a/distrib/sets/sets.subr	Tue Feb 07 18:48:19 2012 +0000
+++ b/distrib/sets/sets.subr	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: sets.subr,v 1.134 2012/02/03 22:18:04 joerg Exp $
+#	$NetBSD: sets.subr,v 1.135 2012/02/07 19:13:24 joerg Exp $
 #
 
 #
@@ -176,7 +176,7 @@
 # In each file, a record consists of a path and a System Package name,
 # separated by whitespace. E.g.,
 #
-# 	# $NetBSD: sets.subr,v 1.134 2012/02/03 22:18:04 joerg Exp $
+# 	# $NetBSD: sets.subr,v 1.135 2012/02/07 19:13:24 joerg Exp $
 # 	.			base-sys-root	[keyword[,...]]
 # 	./altroot		base-sys-root
 # 	./bin			base-sys-root
@@ -228,6 +228,7 @@
 #	lint			${MKLINT} != no
 #	llvm			${MKLLVM} != no
 #	lvm			${MKLVM} != no
+#	makemandb		${MKMAKEMANDB} != no
 #	man			${MKMAN} != no
 #	manpages		${MKMANPAGES} != no
 #	manz			${MKMANZ} != no
--- a/etc/daily	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/daily	Tue Feb 07 19:13:24 2012 +0000
@@ -1,6 +1,6 @@
 #!/bin/sh -
 #
-#	$NetBSD: daily,v 1.78 2011/12/17 12:27:42 darcy Exp $
+#	$NetBSD: daily,v 1.79 2012/02/07 19:13:30 joerg Exp $
 #	@(#)daily	8.2 (Berkeley) 1/25/94
 #
 
@@ -290,6 +290,16 @@
 	fi
 fi
 
+run_makemandb=YES
+
+if checkyesno run_makemandb; then
+	if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+		echo ""
+		echo "Updating man page index:"
+		nice -n 5 /usr/sbin/makemandb
+	fi
+fi
+
 if [ -f /etc/daily.local ]; then
 	( . /etc/daily.local ) > $TMP 2>&1
 	if [ -s $TMP ] ; then
--- a/etc/defaults/daily.conf	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/defaults/daily.conf	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: daily.conf,v 1.15 2011/12/17 12:27:42 darcy Exp $
+#	$NetBSD: daily.conf,v 1.16 2012/02/07 19:13:30 joerg Exp $
 #
 # /etc/defaults/daily.conf --
 #	default configuration of /etc/daily.conf
@@ -25,6 +25,7 @@
 full_netstat=NO
 run_fsck=NO
 run_fsck_flags=""
+run_makemandb=YES
 run_rdist=YES
 run_security=YES
 separate_security_email=YES
--- a/etc/defaults/rc.conf	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/defaults/rc.conf	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: rc.conf,v 1.118 2011/12/23 05:05:04 tsutsui Exp $
+#	$NetBSD: rc.conf,v 1.119 2012/02/07 19:13:30 joerg Exp $
 #
 # /etc/defaults/rc.conf --
 #	default configuration of /etc/rc.conf
@@ -366,3 +366,6 @@
 # Entropy load/save to/from /dev/random at startup/shutdown
 #
 random_seed=YES
+
+# Creating / updating of man page index on boot
+makemandb=YES
--- a/etc/defaults/weekly.conf	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/defaults/weekly.conf	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: weekly.conf,v 1.3 2007/02/15 23:04:10 reed Exp $
+#	$NetBSD: weekly.conf,v 1.4 2012/02/07 19:13:30 joerg Exp $
 #
 # /etc/defaults/weekly.conf --
 #	default configuration of /etc/weekly.conf
@@ -11,3 +11,4 @@
 
 rebuild_locatedb=YES
 rebuild_whatisdb=NO
+rebuild_mandb=YES
--- a/etc/rc.d/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/rc.d/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.87 2011/12/17 13:37:49 tsutsui Exp $
+# $NetBSD: Makefile,v 1.88 2012/02/07 19:13:30 joerg Exp $
 
 .include <bsd.own.mk>
 
@@ -26,7 +26,7 @@
 		irdaattach iscsi_target isdnd isibootd \
 		kdc \
 		ldconfig ldpd local lpd lvm \
-		mdnsd mixerctl mopd motd mountall mountcritlocal \
+		makemandb mdnsd mixerctl mopd motd mountall mountcritlocal \
 		mountcritremote mountd moused mrouted \
 		named ndbootd network newsyslog nfsd nfslocking npf ntpd ntpdate \
 		perusertmp pf pf_boot pflogd postfix powerd ppp pwcheck \
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/etc/rc.d/makemandb	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,24 @@
+#!/bin/sh
+#
+# $NetBSD: makemandb,v 1.1 2012/02/07 19:13:30 joerg Exp $
+#
+
+# PROVIDE: makemandb
+# REQUIRE: DAEMON
+# BEFORE:  LOGIN
+
+$_rc_subr_loaded . /etc/rc.subr
+
+name="makemandb"
+required_files="/etc/man.conf /usr/sbin/makemandb"
+start_cmd="makemandb_start"
+stop_cmd=":"
+
+makemandb_start()
+{
+	# Initiate update build in the background
+	nice -n 5 /usr/sbin/makemandb -q > /dev/null 2>&1 &
+}
+
+load_rc_config $name
+run_rc_command "$1"
--- a/etc/weekly	Tue Feb 07 18:48:19 2012 +0000
+++ b/etc/weekly	Tue Feb 07 19:13:24 2012 +0000
@@ -1,6 +1,6 @@
 #!/bin/sh -
 #
-#	$NetBSD: weekly,v 1.24 2010/12/27 03:38:52 christos Exp $
+#	$NetBSD: weekly,v 1.25 2012/02/07 19:13:30 joerg Exp $
 #	from: @(#)weekly	8.2 (Berkeley) 1/2/94
 #
 
@@ -81,7 +81,8 @@
 
 if checkyesno rebuild_whatisdb; then
 	echo ""
-	if [ -f /etc/man.conf -o -d /usr/share/man ]; then
+	if [ -x /usr/libexec/makewhatis -a \( -f /etc/man.conf -o \
+	     -o -d /usr/share/man \) ]; then
 		echo "Rebuilding whatis databases:"
 		nice -n 5 /usr/libexec/makewhatis -f
 	else
@@ -89,6 +90,16 @@
 	fi
 fi
 
+if checkyesno rebuild_mandb; then
+	echo ""
+	if [ -f /etc/man.conf -a -x /usr/sbin/makemandb ]; then
+		echo "Rebuilding man page index:"
+		nice -n 5 /usr/sbin/makemandb -f
+	else
+		echo "Not rebuilding man page index"
+	fi
+fi
+
 if [ -f /etc/weekly.local ]; then
 	. /etc/weekly.local > $TMP 2>&1
 	if [ -s $TMP ] ; then
--- a/lib/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/lib/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: Makefile,v 1.180 2011/11/05 22:40:01 christos Exp $
+#	$NetBSD: Makefile,v 1.181 2012/02/07 19:13:31 joerg Exp $
 #	from: @(#)Makefile	5.25.1.1 (Berkeley) 5/7/91
 
 .include <bsd.own.mk>
@@ -179,6 +179,8 @@
 SUBDIR+=	../crypto/external/bsd/libsaslc	# depends on heimdal, openssl
 .endif
 
+SUBDIR+=	../external/bsd/mdocml/lib
+
 #==================== 4th library dependency barrier ====================
 SUBDIR+=	.WAIT
 
--- a/libexec/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/libexec/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,12 +1,16 @@
-#	$NetBSD: Makefile,v 1.60 2011/04/15 18:05:45 elric Exp $
+#	$NetBSD: Makefile,v 1.61 2012/02/07 19:13:31 joerg Exp $
 #	@(#)Makefile	8.1 (Berkeley) 6/4/93
 
 .include <bsd.own.mk>
 
 SUBDIR= atrun \
-	comsat fingerd ftpd getNAME getty httpd identd ld.aout_so \
-	ld.elf_so lfs_cleanerd mail.local makekey makewhatis rexecd \
+	comsat fingerd ftpd getty httpd identd ld.aout_so \
+	ld.elf_so lfs_cleanerd mail.local makekey rexecd \
 	rlogind rshd rpc.rquotad rpc.rstatd rpc.rusersd rpc.rwalld \
 	rpc.sprayd talkd telnetd tftpd utmp_update
 
+.if ${MKMAKEMANDB} == "no"
+SUBDIR+= getNAME makewhatis
+.endif
+
 .include <bsd.subdir.mk>
--- a/share/man/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/share/man/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: Makefile,v 1.29 2004/02/09 07:58:01 lukem Exp $
+#	$NetBSD: Makefile,v 1.30 2012/02/07 19:13:31 joerg Exp $
 #	from: @(#)Makefile	8.2 (Berkeley) 4/16/94
 
 .include <bsd.own.mk>
@@ -8,6 +8,7 @@
 
 WHATISDBDIR?=	/usr/share/man
 
+.if ${MKMAKEMANDB} == "no"
 makedb:
 	${TOOL_MAKEWHATIS} ${DESTDIR}${WHATISDBDIR}
 .if ${MKUNPRIVED} != "no"
@@ -18,5 +19,9 @@
 makedb:
 .endif
 
+.else
+makedb:
+.endif
+
 .include <bsd.subdir.mk>
 .include <bsd.sys.mk>
--- a/share/man/man0/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/share/man/man0/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,11 +1,11 @@
 #	from: @(#)Makefile	5.9 (Berkeley) 7/1/91
-#	$NetBSD: Makefile,v 1.16 2008/10/25 22:27:36 apb Exp $
+#	$NetBSD: Makefile,v 1.17 2012/02/07 19:13:31 joerg Exp $
 
 NOOBJ=		# defined
 
 .include <bsd.own.mk>
 
-.if ${MKSHARE} != "no"
+.if ${MKSHARE} != "no" && ${MKMAKEMANDB} == "no"
 FILES=		makewhatis.sed
 FILESDIR=	/usr/share/man
 .endif
--- a/share/mk/bsd.own.mk	Tue Feb 07 18:48:19 2012 +0000
+++ b/share/mk/bsd.own.mk	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: bsd.own.mk,v 1.693 2011/11/25 22:03:42 skrll Exp $
+#	$NetBSD: bsd.own.mk,v 1.694 2012/02/07 19:13:31 joerg Exp $
 
 # This needs to be before bsd.init.mk
 .if defined(BSD_MK_COMPAT_FILE)
@@ -815,6 +815,7 @@
 	MKLDAP MKLINKLIB MKLINT MKLVM \
 	MKMAN \
 	MKMDNS \
+	MKMAKEMANDB \
 	MKNLS \
 	MKNPF \
 	MKOBJ \
--- a/usr.bin/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/usr.bin/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,9 +1,9 @@
-#	$NetBSD: Makefile,v 1.206 2012/01/04 16:25:13 yamt Exp $
+#	$NetBSD: Makefile,v 1.207 2012/02/07 19:13:31 joerg Exp $
 #	from: @(#)Makefile	8.3 (Berkeley) 1/7/94
 
 .include <bsd.own.mk>
 
-SUBDIR= apply apropos asa at audio audiocfg \
+SUBDIR= apply asa at audio audiocfg \
 	banner basename biff bthset btkey btpin \
 	bzip2 bzip2recover cal calendar cap_mkdb cdplay \
 	checknr chflags chpass cksum cmp cleantags col colcrt colrm \
@@ -31,11 +31,15 @@
 	uniq units unvis unzip usbhidaction usbhidctl users utoppya \
 	uudecode uuencode uuidgen vacation vgrind videoctl vis \
 	vmstat vndcompress w \
-	wall wc what whatis whereis who whois window \
+	wall wc what whereis who whois window \
 	write xargs xinstall xlint xstr yes
 
 SUBDIR+= ../external/zlib/pigz/bin/pigz
 
+.if MKMAKEMANDB == "no"
+SUBDIR+= apropos whatis
+.endif
+
 .if (${MKBSDGREP} != "no")
 SUBDIR+= grep
 .endif
--- a/usr.sbin/Makefile	Tue Feb 07 18:48:19 2012 +0000
+++ b/usr.sbin/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -1,4 +1,4 @@
-#	$NetBSD: Makefile,v 1.260 2012/01/04 16:25:14 yamt Exp $
+#	$NetBSD: Makefile,v 1.261 2012/02/07 19:13:32 joerg Exp $
 #	from: @(#)Makefile	5.20 (Berkeley) 6/12/93
 
 .include <bsd.own.mk>
@@ -32,6 +32,10 @@
 	wake wiconfig wlanctl wsconscfg wsfontload wsmoused wsmuxctl \
 	zdump zic
 
+.if ${MKMAKEMANDB} != "no"
+SUBDIR+= makemandb
+.endif
+
 .if (${MKYP} != "no")
 SUBDIR+= rpc.yppasswdd ypbind yppoll ypserv ypset
 .endif
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/DBSCHEMA	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,52 @@
+There are three tables in the database at present:
+
+(1) mandb:
+    This is the main FTS table which contains all the content from 
+    man pages
+
+  COLUMN NAME       DESCRIPTION
+  1. section        Section number of the page  
+  2. name           The name of the page  
+  3. name_desc      The one line description from the NAME section  
+  4. desc           Contains the content from DESCRIPTION 
+                    section and any other section which 
+                    is not indexed in a separate column.
+  5. lib            The LIBRARY section
+  6. return_vals    RETURN VALUES section
+  7. env            ENVIRONMENT section
+  8. files          FILES section
+  9. exit_status    EXIT STATUS section
+  10. diagnostics   DIAGNOSTICS section
+  11. errors        ERRORS section
+  12. md5_hash      md5 hash of the man page (UNIQUE)
+  13. machine       The machine architecture (if any) for which this page is
+                    relevant.
+
+(2) mandb_meta:
+    This table contains md5 hashes of all the indexed man 
+    pages. This is there to make sure we do not index 
+    the same man page twice, i.e. to prevent indexing 
+    hardlinks.
+
+  COLUMN NAME       DESCRIPTION
+  1. device         (dev_t)Logical device number from stat(2)
+  2. inode          (ino_t)Inode number from stat(2)
+  3. mtime          Last modification time from stat(2)
+  4. file           Absolute path name (UNIQUE constraint)
+  5. md5_hash       MD5 Hash of the man page (UNIQUE)
+  6. id             A unique integer ID for the page, which
+                    refers to the docid column in mandb (Implicit foreign 
+                    key?) PRIMARY KEY
+  {device, inode} form a UNIQUE index
+
+(3) mandb_links:
+    This is an index of all the hard/soft links of the man 
+    pages. The list of the linked pages is usually obtained 
+    from the multiple .Nm entries in the page.
+
+  COLUMN NAME       DESCRIPTION
+  1. link           The name of the hard/soft link (index) 
+  2. target         Name of the target page to which it points  
+  3. section        The section number  
+  4. machine        The machine architecture (if any) for which 
+                    the page is relevant
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/Makefile	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,40 @@
+# $NetBSD: Makefile,v 1.1 2012/02/07 19:13:32 joerg Exp $
+
+.include <bsd.own.mk>
+
+MDIST=	${NETBSDSRCDIR}/external/bsd/mdocml/dist
+MDOCDIR=${NETBSDSRCDIR}/external/bsd/mdocml
+
+PROGS=			makemandb apropos whatis
+SRCS.makemandb=		makemandb.c apropos-utils.c
+SRCS.apropos=	apropos.c apropos-utils.c
+SRCS.whatis=	whatis.c apropos-utils.c
+MAN.makemandb=	makemandb.8
+MAN.apropos=	apropos.1
+MAN.whatis=	whatis.1
+
+BINDIR.apropos=		/usr/bin
+BINDIR.makemandb=	/usr/sbin
+BINDIR.whatis=		/usr/bin
+
+CPPFLAGS+=-I${MDIST} -I${.OBJDIR}
+
+MDOCMLOBJDIR!=	cd ${MDOCDIR}/lib/libmandoc && ${PRINTOBJDIR}
+MDOCMLLIB=	${MDOCMLOBJDIR}/libmandoc.a
+
+DPADD.makemandb+= 	${MDOCMLLIB}
+LDADD.makemandb+= 	-L${MDOCMLOBJDIR} -lmandoc
+DPADD+=		${LIBSQLITE3} ${LIBM} ${LIBZ} ${LIBUTIL}
+LDADD+=		-lsqlite3 -lm -lz -lutil
+
+stopwords.c: stopwords.txt
+	( set -e; ${TOOL_NBPERF} -n stopwords_hash -s -p ${.ALLSRC};	\
+	echo 'static const char *stopwords[] = {';			\
+	${TOOL_SED} -e 's|^\(.*\)$$|	"\1",|' ${.ALLSRC};		\
+	echo '};'							\
+	) > ${.TARGET}
+
+DPSRCS+=	stopwords.c
+CLEANFILES+=	stopwords.c
+
+.include <bsd.prog.mk>
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/apropos-utils.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,67 @@
+.\" $NetBSD: apropos-utils.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt APROPOS-UTILS 3
+.Os
+.Sh NAME
+.Nm apropos-utils
+.Nd support functions for querying the man page index
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Ft int
+.Fn run_query "sqlite3 *db" "const char *snippet_args[3]" "query_args *args"
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+These functions all live in the
+.Pa apropos-utils.h
+header file.
+They operate on
+.Pa /var/db/man.db
+which is an Sqlite database containing a full text search index of the manual
+pages.
+The functions provide an easy to use interface to query the database and
+develop applications on top of it.
+.Sh SEE ALSO
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/apropos-utils.c	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,755 @@
+/*	$NetBSD	*/
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD: apropos-utils.c,v 1.1 2012/02/07 19:13:32 joerg Exp $");
+
+#include <sys/stat.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <err.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <util.h>
+#include <zlib.h>
+
+#include "apropos-utils.h"
+#include "mandoc.h"
+#include "sqlite3.h"
+
+typedef struct orig_callback_data {
+	void *data;
+	int (*callback) (void *, const char *, const char *, const char *,
+		const char *, size_t);
+} orig_callback_data;
+
+typedef struct inverse_document_frequency {
+	double value;
+	int status;
+} inverse_document_frequency;
+
+/* weights for individual columns */
+static const double col_weights[] = {
+	2.0,	// NAME
+	2.00,	// Name-description
+	0.55,	// DESCRIPTION
+	0.10,	// LIBRARY
+	0.001,	//RETURN VALUES
+	0.20,	//ENVIRONMENT
+	0.01,	//FILES
+	0.001,	//EXIT STATUS
+	2.00,	//DIAGNOSTICS
+	0.05,	//ERRORS
+	0.00,	//md5_hash
+	1.00	//machine
+};
+
+/*
+ * lower --
+ *  Converts the string str to lower case
+ */
+char *
+lower(char *str)
+{
+	assert(str);
+	int i = 0;
+	char c;
+	while (str[i] != '\0') {
+		c = tolower((unsigned char) str[i]);
+		str[i++] = c;
+	}
+	return str;
+}
+
+/*
+* concat--
+*  Utility function. Concatenates together: dst, a space character and src. 
+* dst + " " + src 
+*/
+void
+concat(char **dst, const char *src)
+{
+	concat2(dst, src, strlen(src));
+}
+
+void
+concat2(char **dst, const char *src, size_t srclen)
+{
+	size_t total_len, dst_len;
+	assert(src != NULL);
+
+	/* If destination buffer dst is NULL, then simply strdup the source buffer */
+	if (*dst == NULL) {
+		*dst = estrdup(src);
+		return;
+	}
+
+	dst_len = strlen(*dst);
+	/*
+	 * NUL Byte and separator space
+	 */
+	total_len = dst_len + srclen + 2;
+
+	*dst = erealloc(*dst, total_len);
+
+	/* Append a space at the end of dst */
+	(*dst)[dst_len++] = ' ';
+
+	/* Now, copy src at the end of dst */	
+	memcpy(*dst + dst_len, src, srclen + 1);
+}
+
+void
+close_db(sqlite3 *db)
+{
+	sqlite3_close(db);
+	sqlite3_shutdown();
+}
+
+/*
+ * create_db --
+ *  Creates the database schema.
+ */
+static int
+create_db(sqlite3 *db)
+{
+	const char *sqlstr = NULL;
+	char *schemasql;
+	char *errmsg = NULL;
+	
+/*------------------------ Create the tables------------------------------*/
+
+#if NOTYET
+	sqlite3_exec(db, "PRAGMA journal_mode = WAL", NULL, NULL, NULL);
+#else
+	sqlite3_exec(db, "PRAGMA journal_mode = DELETE", NULL, NULL, NULL);
+#endif
+
+	schemasql = sqlite3_mprintf("PRAGMA user_version = %d",
+	    APROPOS_SCHEMA_VERSION);
+	sqlite3_exec(db, schemasql, NULL, NULL, &errmsg);
+	if (errmsg != NULL)
+		goto out;
+	sqlite3_free(schemasql);
+
+	sqlstr = "CREATE VIRTUAL TABLE mandb USING fts4(section, name, "
+			    "name_desc, desc, lib, return_vals, env, files, "
+			    "exit_status, diagnostics, errors, md5_hash UNIQUE, machine, "
+			    "compress=zip, uncompress=unzip, tokenize=porter); "	//mandb
+			"CREATE TABLE IF NOT EXISTS mandb_meta(device, inode, mtime, "
+			    "file UNIQUE, md5_hash UNIQUE, id  INTEGER PRIMARY KEY); "
+				//mandb_meta
+			"CREATE TABLE IF NOT EXISTS mandb_links(link, target, section, "
+			    "machine); ";	//mandb_links
+
+	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+	if (errmsg != NULL)
+		goto out;
+
+	sqlstr = "CREATE INDEX IF NOT EXISTS index_mandb_links ON mandb_links "
+			"(link); "
+			"CREATE INDEX IF NOT EXISTS index_mandb_meta_dev ON mandb_meta "
+			"(device, inode)";
+	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+	if (errmsg != NULL)
+		goto out;
+	return 0;
+
+out:
+	warnx("%s", errmsg);
+	free(errmsg);
+	sqlite3_close(db);
+	sqlite3_shutdown();
+	return -1;
+}
+
+/*
+ * zip --
+ *  User defined Sqlite function to compress the FTS table
+ */
+static void
+zip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{	
+	int nin;
+	long int nout;
+	const unsigned char * inbuf;
+	unsigned char *outbuf;
+
+	assert(nval == 1);
+	nin = sqlite3_value_bytes(apval[0]);
+	inbuf = (const unsigned char *) sqlite3_value_blob(apval[0]);
+	nout = nin + 13 + (nin + 999) / 1000;
+	outbuf = emalloc(nout);
+	compress(outbuf, (unsigned long *) &nout, inbuf, nin);
+	sqlite3_result_blob(pctx, outbuf, nout, free);
+}
+
+/*
+ * unzip --
+ *  User defined Sqlite function to uncompress the FTS table.
+ */
+static void
+unzip(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+	unsigned int rc;
+	unsigned char *outbuf;
+	z_stream stream;
+
+	assert(nval == 1);
+	stream.next_in = __UNCONST(sqlite3_value_blob(apval[0]));
+	stream.avail_in = sqlite3_value_bytes(apval[0]);
+	stream.avail_out = stream.avail_in * 2 + 100;
+	stream.next_out = outbuf = emalloc(stream.avail_out);
+	stream.zalloc = NULL;
+	stream.zfree = NULL;
+
+	if (inflateInit(&stream) != Z_OK) {
+		free(outbuf);
+		return;
+	}
+
+	while ((rc = inflate(&stream, Z_SYNC_FLUSH)) != Z_STREAM_END) {
+		if (rc != Z_OK ||
+		    (stream.avail_out != 0 && stream.avail_in == 0)) {
+			free(outbuf);
+			return;
+		}
+		outbuf = erealloc(outbuf, stream.total_out * 2);
+		stream.next_out = outbuf + stream.total_out;
+		stream.avail_out = stream.total_out;
+	}
+	if (inflateEnd(&stream) != Z_OK) {
+		free(outbuf);
+		return;
+	}
+	outbuf = erealloc(outbuf, stream.total_out);
+	sqlite3_result_text(pctx, (const char *) outbuf, stream.total_out, free);
+}
+
+/* init_db --
+ *   Prepare the database. Register the compress/uncompress functions and the
+ *   stopword tokenizer.
+ *	 db_flag specifies the mode in which to open the database. 3 options are 
+ *   available:
+ *   	1. DB_READONLY: Open in READONLY mode. An error if db does not exist.
+ *  	2. DB_READWRITE: Open in read-write mode. An error if db does not exist.
+ *  	3. DB_CREATE: Open in read-write mode. It will try to create the db if
+ *			it does not exist already.
+ *  RETURN VALUES:
+ *		The function will return NULL in case the db does not exist and DB_CREATE 
+ *  	was not specified. And in case DB_CREATE was specified and yet NULL is 
+ *  	returned, then there was some other error.
+ *  	In normal cases the function should return a handle to the db.
+ */
+sqlite3 *
+init_db(int db_flag)
+{
+	sqlite3 *db = NULL;
+	sqlite3_stmt *stmt;
+	struct stat sb;
+	int rc;
+	int create_db_flag = 0;
+
+	/* Check if the database exists or not */
+	if (!(stat(DBPATH, &sb) == 0 && S_ISREG(sb.st_mode))) {
+		/* Database does not exist, check if DB_CREATE was specified, and set
+		 * flag to create the database schema
+		 */
+		if (db_flag != (MANDB_CREATE)) {
+			warnx("Missing apropos database. "
+			      "Please run makemandb to create it.");
+			return NULL;
+		}
+		create_db_flag = 1;
+	}
+
+	/* Now initialize the database connection */
+	sqlite3_initialize();
+	rc = sqlite3_open_v2(DBPATH, &db, db_flag, NULL);
+	
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_shutdown();
+		return NULL;
+	}
+
+	if (create_db_flag && create_db(db) < 0) {
+		warnx("%s", "Unable to create database schema");
+		goto error;
+	}
+
+	rc = sqlite3_prepare_v2(db, "PRAGMA user_version", -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("Unable to query schema version");
+		goto error;
+	}
+	if (sqlite3_step(stmt) != SQLITE_ROW) {
+		sqlite3_finalize(stmt);
+		warnx("Unable to query schema version");
+		goto error;
+	}
+	if (sqlite3_column_int(stmt, 0) != APROPOS_SCHEMA_VERSION) {
+		sqlite3_finalize(stmt);
+		warnx("Incorrect schema version found. "
+		      "Please run makemandb -f.");
+		goto error;
+	}
+	sqlite3_finalize(stmt);
+
+	sqlite3_extended_result_codes(db, 1);
+	
+	/* Register the zip and unzip functions for FTS compression */
+	rc = sqlite3_create_function(db, "zip", 1, SQLITE_ANY, NULL, zip, NULL, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("Unable to register function: compress");
+		goto error;
+	}
+
+	rc = sqlite3_create_function(db, "unzip", 1, SQLITE_ANY, NULL, 
+                                 unzip, NULL, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("Unable to register function: uncompress");
+		goto error;
+	}
+	return db;
+error:
+	sqlite3_close(db);
+	sqlite3_shutdown();
+	return NULL;
+}
+
+/*
+ * rank_func --
+ *  Sqlite user defined function for ranking the documents.
+ *  For each phrase of the query, it computes the tf and idf and adds them over.
+ *  It computes the final rank, by multiplying tf and idf together.
+ *  Weight of term t for document d = (term frequency of t in d * 
+ *                                      inverse document frequency of t) 
+ *
+ *  Term Frequency of term t in document d = Number of times t occurs in d / 
+ *	                                        Number of times t appears in all 
+ *											documents
+ *
+ *  Inverse document frequency of t = log(Total number of documents / 
+ *										Number of documents in which t occurs)
+ */
+static void
+rank_func(sqlite3_context *pctx, int nval, sqlite3_value **apval)
+{
+	inverse_document_frequency *idf = sqlite3_user_data(pctx);
+	double tf = 0.0;
+	const unsigned int *matchinfo;
+	int ncol;
+	int nphrase;
+	int iphrase;
+	int ndoc;
+	int doclen = 0;
+	const double k = 3.75;
+	/* Check that the number of arguments passed to this function is correct. */
+	assert(nval == 1);
+
+	matchinfo = (const unsigned int *) sqlite3_value_blob(apval[0]);
+	nphrase = matchinfo[0];
+	ncol = matchinfo[1];
+	ndoc = matchinfo[2 + 3 * ncol * nphrase + ncol];
+	for (iphrase = 0; iphrase < nphrase; iphrase++) {
+		int icol;
+		const unsigned int *phraseinfo = &matchinfo[2 + ncol+ iphrase * ncol * 3];
+		for(icol = 1; icol < ncol; icol++) {
+			
+			/* nhitcount: number of times the current phrase occurs in the current
+			 *            column in the current document.
+			 * nglobalhitcount: number of times current phrase occurs in the current
+			 *                  column in all documents.
+			 * ndocshitcount:   number of documents in which the current phrase 
+			 *                  occurs in the current column at least once.
+			 */
+  			int nhitcount = phraseinfo[3 * icol];
+			int nglobalhitcount = phraseinfo[3 * icol + 1];
+			int ndocshitcount = phraseinfo[3 * icol + 2];
+			doclen = matchinfo[2 + icol ];
+			double weight = col_weights[icol - 1];
+			if (idf->status == 0 && ndocshitcount)
+				idf->value += log(((double)ndoc / ndocshitcount))* weight;
+
+			/* Dividing the tf by document length to normalize the effect of 
+			 * longer documents.
+			 */
+			if (nglobalhitcount > 0 && nhitcount)
+				tf += (((double)nhitcount  * weight) / (nglobalhitcount * doclen));
+		}
+	}
+	idf->status = 1;
+	
+	/* Final score = (tf * idf)/ ( k + tf)
+	 *	Dividing by k+ tf further normalizes the weight leading to better 
+	 *  results.
+	 *  The value of k is experimental
+	 */
+	double score = (tf * idf->value/ ( k + tf)) ;
+	sqlite3_result_double(pctx, score);
+	return;
+}
+
+/*
+ *  run_query --
+ *  Performs the searches for the keywords entered by the user.
+ *  The 2nd param: snippet_args is an array of strings providing values for the
+ *  last three parameters to the snippet function of sqlite. (Look at the docs).
+ *  The 3rd param: args contains rest of the search parameters. Look at 
+ *  arpopos-utils.h for the description of individual fields.
+ *  
+ */
+int
+run_query(sqlite3 *db, const char *snippet_args[3], query_args *args)
+{
+	const char *default_snippet_args[3];
+	char *section_clause = NULL;
+	char *limit_clause = NULL;
+	char *machine_clause = NULL;
+	char *query;
+	const char *section;
+	char *name;
+	const char *name_desc;
+	const char *machine;
+	const char *snippet;
+	char *m = NULL;
+	int rc;
+	inverse_document_frequency idf = {0, 0};
+	sqlite3_stmt *stmt;
+
+	if (args->machine)
+		easprintf(&machine_clause, "AND machine = \'%s\' ", args->machine);
+
+	/* Register the rank function */
+	rc = sqlite3_create_function(db, "rank_func", 1, SQLITE_ANY, (void *)&idf, 
+	                             rank_func, NULL, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_close(db);
+		sqlite3_shutdown();
+		errx(EXIT_FAILURE, "Unable to register the ranking function");
+	}
+	
+	/* We want to build a query of the form: "select x,y,z from mandb where
+	 * mandb match :query [AND (section LIKE '1' OR section LIKE '2' OR...)]
+	 * ORDER BY rank DESC..."
+	 * NOTES: 1. The portion in square brackets is optional, it will be there 
+	 * only if the user has specified an option on the command line to search in 
+	 * one or more specific sections.
+	 * 2. I am using LIKE operator because '=' or IN operators do not seem to be
+	 * working with the compression option enabled.
+	 */
+
+	if (args->sec_nums) {
+		char *temp;
+		int i;
+
+		for (i = 0; i < SECMAX; i++) {
+			if (args->sec_nums[i] == 0)
+				continue;
+			easprintf(&temp, " OR section = \'%d\'", i + 1);
+			if (section_clause) {
+				concat(&section_clause, temp);
+				free(temp);
+			} else {
+				section_clause = temp;
+			}
+		}
+		if (section_clause) {
+			/*
+			 * At least one section requested, add glue for query.
+			 */
+			temp = section_clause;
+			/* Skip " OR " before first term. */
+			easprintf(&section_clause, " AND (%s)", temp + 4);
+			free(temp);
+		}
+	}
+	if (args->nrec >= 0) {
+		/* Use the provided number of records and offset */
+		easprintf(&limit_clause, " LIMIT %d OFFSET %d",
+		    args->nrec, args->offset);
+	}
+
+	if (snippet_args == NULL) {
+		default_snippet_args[0] = "";
+		default_snippet_args[1] = "";
+		default_snippet_args[2] = "...";
+		snippet_args = default_snippet_args;
+	}
+	query = sqlite3_mprintf("SELECT section, name, name_desc, machine,"
+	    " snippet(mandb, %Q, %Q, %Q, -1, 40 ),"
+	    " rank_func(matchinfo(mandb, \"pclxn\")) AS rank"
+	    " FROM mandb"
+	    " WHERE mandb MATCH %Q %s "
+	    "%s"
+	    " ORDER BY rank DESC"
+	    "%s",
+	    snippet_args[0], snippet_args[1], snippet_args[2], args->search_str,
+	    machine_clause ? machine_clause : "",
+	    section_clause ? section_clause : "",
+	    limit_clause ? limit_clause : "");
+
+	free(machine_clause);
+	free(section_clause);
+	free(limit_clause);
+
+	if (query == NULL) {
+		*args->errmsg = estrdup("malloc failed");
+		return -1;
+	}
+	rc = sqlite3_prepare_v2(db, query, -1, &stmt, NULL);
+	if (rc == SQLITE_IOERR) {
+		warnx("Corrupt database. Please rerun makemandb");
+		sqlite3_free(query);
+		return -1;
+	} else if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_free(query);
+		return -1;
+	}
+
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		section = (const char *) sqlite3_column_text(stmt, 0);
+		name_desc = (const char *) sqlite3_column_text(stmt, 2);
+		machine = (const char *) sqlite3_column_text(stmt, 3);
+		snippet = (const char *) sqlite3_column_text(stmt, 4);
+		if (machine && machine[0]) {
+			m = estrdup(machine);
+			easprintf(&name, "%s/%s", lower(m),
+				sqlite3_column_text(stmt, 1));
+			free(m);
+		} else {
+			name = estrdup((const char *) sqlite3_column_text(stmt, 1));
+		}
+
+		(args->callback)(args->callback_data, section, name, name_desc, snippet,
+			strlen(snippet));
+
+		free(name);
+	}
+
+	sqlite3_finalize(stmt);
+	sqlite3_free(query);
+	return *(args->errmsg) == NULL ? 0 : -1;
+}
+
+/*
+ * callback_html --
+ *  Callback function for run_query_html. It builds the html output and then
+ *  calls the actual user supplied callback function.
+ */
+static int
+callback_html(void *data, const char *section, const char *name,
+	const char *name_desc, const char *snippet, size_t snippet_length)
+{
+	const char *temp = snippet;
+	int i = 0;
+	size_t sz = 0;
+	int count = 0;
+	struct orig_callback_data *orig_data = (struct orig_callback_data *) data;
+	int (*callback) (void *, const char *, const char *, const char *, 
+		const char *, size_t) = orig_data->callback;
+
+	/* First scan the snippet to find out the number of occurrences of {'>', '<'
+	 * '"', '&'}.
+	 * Then allocate a new buffer with sufficient space to be able to store the
+	 * quoted versions of the special characters {&gt;, &lt;, &quot;, &amp;}.
+	 * Copy over the characters from the original snippet to this buffer while
+	 * replacing the special characters with their quoted versions.
+	 */
+
+	while (*temp) {
+		sz = strcspn(temp, "<>\"&\002\003");
+		temp += sz + 1;
+		count++;
+	}
+	size_t qsnippet_length = snippet_length + count * 5;
+	char *qsnippet = emalloc(qsnippet_length + 1);
+	sz = 0;
+	while (*snippet) {
+		sz = strcspn(snippet, "<>\"&\002\003");
+		if (sz) {
+			memcpy(&qsnippet[i], snippet, sz);
+			snippet += sz;
+			i += sz;
+		}
+
+		switch (*snippet++) {
+		case '<':
+			memcpy(&qsnippet[i], "&lt;", 4);
+			i += 4;
+			break;
+		case '>':
+			memcpy(&qsnippet[i], "&gt;", 4);
+			i += 4;
+			break;
+		case '\"':
+			memcpy(&qsnippet[i], "&quot;", 6);
+			i += 6;
+			break;
+		case '&':
+			/* Don't perform the quoting if this & is part of an mdoc escape
+			 * sequence, e.g. \&
+			 */
+			if (i && *(snippet - 2) != '\\') {
+				memcpy(&qsnippet[i], "&amp;", 5);
+				i += 5;
+			} else {
+				qsnippet[i++] = '&';
+			}
+			break;
+		case '\002':
+			memcpy(&qsnippet[i], "<b>", 3);
+			i += 3;
+			break;
+		case '\003':
+			memcpy(&qsnippet[i], "</b>", 4);
+			i += 4;
+			break;
+		default:
+			break;
+		}
+	}
+	qsnippet[++i] = 0;
+	(*callback)(orig_data->data, section, name, name_desc,
+		(const char *)qsnippet,	qsnippet_length);
+	free(qsnippet);
+	return 0;
+}
+
+/*
+ * run_query_html --
+ *  Utility function to output query result in HTML format.
+ *  It internally calls run_query only, but it first passes the output to it's 
+ *  own custom callback function, which preprocess the snippet for quoting
+ *  inline HTML fragments.
+ *  After that it delegates the call the actual user supplied callback function.
+ */
+int
+run_query_html(sqlite3 *db, query_args *args)
+{
+	struct orig_callback_data orig_data;
+	orig_data.callback = args->callback;
+	orig_data.data = args->callback_data;
+	const char *snippet_args[] = {"\002", "\003", "..."};
+	args->callback = &callback_html;
+	args->callback_data = (void *) &orig_data;
+	return run_query(db, snippet_args, args);
+}
+
+/*
+ * callback_pager --
+ *  A callback similar to callback_html. It overstrikes the matching text in
+ *  the snippet so that it appears emboldened when viewed using a pager like
+ *  more or less.
+ */
+static int
+callback_pager(void *data, const char *section, const char *name, 
+	const char *name_desc, const char *snippet, size_t snippet_length)
+{
+	struct orig_callback_data *orig_data = (struct orig_callback_data *) data;
+	char *psnippet;
+	const char *temp = snippet;
+	int count = 0;
+	int i = 0;
+	size_t sz = 0;
+	size_t psnippet_length;
+
+	/* Count the number of bytes of matching text. For each of these bytes we
+	 * will use 2 extra bytes to overstrike it so that it appears bold when
+	 * viewed using a pager.
+	 */
+	while (*temp) {
+		sz = strcspn(temp, "\002\003");
+		temp += sz;
+		if (*temp == '\003') {
+			count += 2 * (sz);
+		}
+		temp++;
+	}
+
+	psnippet_length = snippet_length + count;
+	psnippet = emalloc(psnippet_length + 1);
+
+	/* Copy the bytes from snippet to psnippet:
+	 * 1. Copy the bytes before \002 as it is.
+	 * 2. The bytes after \002 need to be overstriked till we encounter \003.
+	 * 3. To overstrike a byte 'A' we need to write 'A\bA'
+	 */
+	while (*snippet) {
+		sz = strcspn(snippet, "\002");
+		memcpy(&psnippet[i], snippet, sz);
+		snippet += sz;
+		i += sz;
+
+		/* Don't change this. Advancing the pointer without reading the byte
+		 * is causing strange behavior.
+		 */
+		if (*snippet == '\002')
+			snippet++;
+		while (*snippet && *snippet != '\003') {
+			psnippet[i++] = *snippet;
+			psnippet[i++] = '\b';
+			psnippet[i++] = *snippet++;
+		}
+		if (*snippet)
+			snippet++;
+	}
+
+	psnippet[i] = 0;
+	(orig_data->callback)(orig_data->data, section, name, name_desc, psnippet,
+		psnippet_length);
+	free(psnippet);
+	return 0;
+}
+
+/*
+ * run_query_pager --
+ *  Utility function similar to run_query_html. This function tries to
+ *  pre-process the result assuming it will be piped to a pager.
+ *  For this purpose it first calls it's own callback function callback_pager
+ *  which then delegates the call to the user supplied callback.
+ */
+int run_query_pager(sqlite3 *db, query_args *args)
+{
+	struct orig_callback_data orig_data;
+	orig_data.callback = args->callback;
+	orig_data.data = args->callback_data;
+	const char *snippet_args[] = {"\002", "\003", "..."};
+	args->callback = &callback_pager;
+	args->callback_data = (void *) &orig_data;
+	return run_query(db, snippet_args, args);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/apropos-utils.h	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,92 @@
+/*	$NetBSD	*/
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef APROPOS_UTILS_H
+#define APROPOS_UTILS_H
+
+#include "sqlite3.h"
+
+#define DBPATH "/var/db/man.db"
+#define SECMAX 9
+
+/* Flags for opening the database */
+#define MANDB_READONLY SQLITE_OPEN_READONLY
+#define MANDB_WRITE SQLITE_OPEN_READWRITE
+#define MANDB_CREATE SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE
+
+#define APROPOS_SCHEMA_VERSION 20120130
+
+/*
+ * Used to identify the section of a man(7) page.
+ * This is similar to the enum mdoc_sec defined in mdoc.h from mdocml project.
+ */
+enum man_sec {
+	MANSEC_NAME = 0,
+	MANSEC_SYNOPSIS,
+	MANSEC_LIBRARY,
+	MANSEC_ERRORS,
+	MANSEC_FILES,
+	MANSEC_RETURN_VALUES,
+	MANSEC_EXIT_STATUS,
+	MANSEC_DESCRIPTION,
+	MANSEC_ENVIRONMENT,
+	MANSEC_DIAGNOSTICS,
+	MANSEC_EXAMPLES,
+	MANSEC_STANDARDS,
+	MANSEC_HISTORY,
+	MANSEC_BUGS,
+	MANSEC_AUTHORS,
+	MANSEC_COPYRIGHT,
+	MANSEC_NONE
+};
+
+typedef struct query_args {
+	const char *search_str;		// user query
+	int *sec_nums;		// Section in which to do the search
+	int nrec;			// number of records to fetch
+	int offset;		//From which position to start processing the records
+	const char *machine;
+	int (*callback) (void *, const char *, const char *, const char *,
+		const char *, size_t);	// The callback function
+	void *callback_data;	// data to pass to the callback function
+	char **errmsg;		// buffer for storing the error msg
+} query_args;
+
+char *lower(char *);
+void concat(char **, const char *);
+void concat2(char **, const char *, size_t);
+sqlite3 *init_db(int);
+void close_db(sqlite3 *);
+int run_query(sqlite3 *, const char *[3], query_args *);
+int run_query_html(sqlite3 *, query_args *);
+int run_query_pager(sqlite3 *, query_args *);
+#endif 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/apropos.1	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,107 @@
+.\" $NetBSD: apropos.1,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 6, 2012
+.Dt APROPOS 1
+.Os
+.Sh NAME
+.Nm apropos
+.Nd search the complete content of all man pages
+.Sh SYNOPSIS
+.Nm
+.Op Fl 123456789Ccp
+.Op Fl n Ar Number of results
+.Op Fl S Ar machine
+.Ar query
+.Sh DESCRIPTION
+The
+.Nm
+utility performs a full text search over the complete content of all man pages.
+It uses the FTS engine of Sqlite to perform the search.
+The database is created with the help of
+.Xr makemandb 1
+utility.
+.Pp
+This implementation of the
+.Nm
+utility is more sophisticated than the classical version of
+.Nm .
+Like modern search applications, it uses advanced techniques like stemming
+and term weighting to rank the matching results in decreasing order of
+relevance.
+By default
+.Nm
+will only display the top 10 matches in the output.
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl 1
+Search only within section 1 manual pages.
+.It Fl 2
+Search only within section 2 manual pages.
+.It Fl 3
+Search only within section 3 manual pages.
+.It Fl 4
+Search only within section 4 manual pages.
+.It Fl 5
+Search only within section 5 manual pages.
+.It Fl 6
+Search only within section 6 manual pages.
+.It Fl 7
+Search only within section 7 manual pages.
+.It Fl 8
+Search only within section 8 manual pages.
+.It Fl 9
+Search only within section 9 manual pages.
+.It Fl C
+Do not show the context of the match.
+.It Fl c
+Do show the context of the match (default).
+.It Fl n
+Output up to the specified number of search results.
+The default limit is 10.
+.It Fl p
+Display all matching results and pipe them through a pager (defaulting to
+.Xr more 1 ) .
+.It Fl S
+Limit the search to the pages for the specified machine architecture.
+By default pages for all architectures are shown in the search results.
+.El
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr makemandb 1 ,
+.Xr man 1
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/apropos.c	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,277 @@
+/*	$NetBSD	*/
+/*-
+ * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+ * All rights reserved.
+ *
+ * This code was developed as part of Google's Summer of Code 2011 program.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD: apropos.c,v 1.1 2012/02/07 19:13:32 joerg Exp $");
+
+#include <err.h>
+#include <search.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "sqlite3.h"
+
+typedef struct apropos_flags {
+	int sec_nums[SECMAX];
+	int nresults;
+	int pager;
+	int no_context;
+	const char *machine;
+} apropos_flags;
+
+typedef struct callback_data {
+	int count;
+	FILE *out;
+	apropos_flags *aflags;
+} callback_data;
+
+static char *remove_stopwords(const char *);
+static int query_callback(void *, const char * , const char *, const char *,
+	const char *, size_t);
+__dead static void usage(void);
+
+#define _PATH_PAGER	"/usr/bin/more -s"
+
+int
+main(int argc, char *argv[])
+{
+#ifdef NOTYET
+	static const char *snippet_args[] = {"\033[1m", "\033[0m", "..."};
+#endif
+	query_args args;
+	char *query = NULL;	// the user query
+	char ch;
+	char *errmsg = NULL;
+	char *str;
+	int rc = 0;
+	callback_data cbdata;
+	cbdata.out = stdout;		// the default output stream
+	cbdata.count = 0;
+	apropos_flags aflags;
+	cbdata.aflags = &aflags;
+	sqlite3 *db;
+	setprogname(argv[0]);
+	if (argc < 2)
+		usage();
+
+	memset(&aflags, 0, sizeof(aflags));
+	
+	/*If the user specifies a section number as an option, the corresponding 
+	 * index element in sec_nums is set to the string representing that 
+	 * section number.
+	 */
+	while ((ch = getopt(argc, argv, "123456789Ccn:pS:")) != -1) {
+		switch (ch) {
+		case '1':
+		case '2':
+		case '3':
+		case '4':
+		case '5':
+		case '6':
+		case '7':
+		case '8':
+		case '9':
+			aflags.sec_nums[atoi(&ch) - 1] = 1;
+			break;
+		case 'C':
+			aflags.no_context = 1;
+			break;
+		case 'c':
+			aflags.no_context = 0;
+			break;
+		case 'n':
+			aflags.nresults = atoi(optarg);
+			break;
+		case 'p':	//user wants to view more than 10 results and page them
+			aflags.pager = 1;
+			aflags.nresults = -1;	// Fetch all records
+			break;
+		case 'S':
+			aflags.machine = optarg;
+			break;
+		case '?':
+		default:
+			usage();
+		}
+	}
+	
+	argc -= optind;
+	argv += optind;
+	
+	if (!argc)
+		usage();
+
+	str = NULL;
+	while (argc--)
+		concat(&str, *argv++);
+	/* Eliminate any stopwords from the query */
+	query = remove_stopwords(lower(str));
+	free(str);
+
+	/* if any error occured in remove_stopwords, exit */
+	if (query == NULL)
+		errx(EXIT_FAILURE, "Try using more relevant keywords");
+
+	if ((db = init_db(MANDB_READONLY)) == NULL)
+		exit(EXIT_FAILURE);
+
+	/* If user wants to page the output, then set some settings */
+	if (aflags.pager) {
+		const char *pager = getenv("PAGER");
+		if (pager == NULL)
+			pager = _PATH_PAGER;
+		/* Open a pipe to the pager */
+		if ((cbdata.out = popen(pager, "w")) == NULL) {
+			close_db(db);
+			err(EXIT_FAILURE, "pipe failed");
+		}
+	}
+
+	args.search_str = query;
+	args.sec_nums = aflags.sec_nums;
+	args.nrec = aflags.nresults ? aflags.nresults : 10;
+	args.offset = 0;
+	args.machine = aflags.machine;
+	args.callback = &query_callback;
+	args.callback_data = &cbdata;
+	args.errmsg = &errmsg;
+
+#ifdef NOTYET
+	rc = run_query(db, snippet_args, &args);
+#else
+	rc = run_query_pager(db, &args);
+#endif
+
+	free(query);
+	close_db(db);
+	if (errmsg) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		exit(EXIT_FAILURE);
+	}
+
+	if (rc < 0) {
+		/* Something wrong with the database. Exit */
+		exit(EXIT_FAILURE);
+	}
+	
+	if (cbdata.count == 0) {
+		warnx("No relevant results obtained.\n"
+			  "Please make sure that you spelled all the terms correctly\n"
+			  "Or try using better keywords.");
+	}
+	return 0;
+}
+
+/*
+ * query_callback --
+ *  Callback function for run_query.
+ *  It simply outputs the results from do_query. If the user specified the -p
+ *  option, then the output is sent to a pager, otherwise stdout is the default
+ *  output stream.
+ */
+static int
+query_callback(void *data, const char *section, const char *name,
+	const char *name_desc, const char *snippet, size_t snippet_length)
+{
+	callback_data *cbdata = (callback_data *) data;
+	FILE *out = cbdata->out;
+	cbdata->count++;
+	fprintf(out, "%s(%s)\t%s\n", name, section, name_desc);
+
+	if (cbdata->aflags->no_context == 0)
+		fprintf(out, "%s\n\n", snippet);
+
+	return 0;
+}
+
+#include "stopwords.c"
+
+/*
+ * remove_stopwords--
+ *  Scans the query and removes any stop words from it.
+ *  Returns the modified query or NULL, if it contained only stop words.
+ */
+
+static char *
+remove_stopwords(const char *query)
+{
+	size_t len, idx;
+	char *output, *buf;
+	const char *sep, *next;
+
+	output = buf = emalloc(strlen(query) + 1);
+
+	for (; query[0] != '\0'; query = next) {
+		sep = strchr(query, ' ');
+		if (sep == NULL) {
+			len = strlen(query);
+			next = query + len;
+		} else {
+			len = sep - query;
+			next = sep + 1;
+		}
+		if (len == 0)
+			continue;
+		idx = stopwords_hash(query, len);
+		if (memcmp(stopwords[idx], query, len) == 0 &&
+		    stopwords[idx][len] == '\0')
+			continue;
+		memcpy(buf, query, len);
+		buf += len;
+		*buf++ = ' ';
+	}
+
+	if (output == buf) {
+		free(output);
+		return NULL;
+	}
+	buf[-1] = '\0';
+	return output;
+}
+
+/*
+ * usage --
+ *	print usage message and die
+ */
+static void
+usage(void)
+{
+	fprintf(stderr,
+		"Usage: %s [-n Number of records] [-p] [-123456789] [-S machine] query\n",
+		getprogname());
+	exit(1);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/close_db.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,62 @@
+.\" $NetBSD: close_db.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt CLOSE_DB 3
+.Os
+.Sh NAME
+.Nm close_db
+.Nd close apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft void
+.Fn close_db "sqlite3 *db"
+.Sh DESCRIPTION
+The
+.Fn close_db
+function will close the connection to
+.Pa /var/db/man.db
+referenced by
+.Fa db
+and release any resources being used by it safely.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/init_db.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,92 @@
+.\" $NetBSD: init_db.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd August 17, 2011
+.Dt INIT_DB 3
+.Os
+.Sh NAME
+.Nm init_db
+.Nd open apropos database connection
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft sqlite3 *
+.Fn init_db "int db_flag"
+.Sh DESCRIPTION
+The
+.Fn init_db
+function will try to initialize a connection to
+.Pa man.db
+which is an
+Sqlite database and return a pointer to a sqlite3 structure.
+The connection should subsequentially be closed by calling
+.Fn close_db
+which will also release any resources being used by it.
+.Pp
+The argument
+.Fa db_flag
+can be one of the following:
+.Bl -hang -width -compact
+.It Dv MANDB_READONLY
+This will open the database in read-only mode.
+Use this when you only want to query the database.
+.It Dv MANDB_WRITE
+This will open the database in read/write mode.
+.It Dv MANDB_CREATE
+This will open the database in read/write mode, and will also create
+the database schema if it does not exist already.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn init_db
+function will return a pointer to a sqlite3 structure which represents
+a connection to the database.
+.Pp
+In case the file
+.Pa /var/db/man.db
+does not exist and
+.Dv DB_CREATE
+is not used as a value of
+.Fa db_flag ,
+.Dv NULL
+will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/makemandb.8	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,109 @@
+.\" $NetBSD: makemandb.8,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 4, 2012
+.Dt MAKEMANDB 8
+.Os
+.Sh NAME
+.Nm makemandb
+.Nd parse the manual pages and build a search index over them
+.Sh SYNOPSIS
+.Nm
+.Op Fl C Ar path
+.Op Fl floqv
+.Sh DESCRIPTION
+The
+.Nm
+utility traverses the directories containing man pages, parses the manual
+pages with the help of libmandoc and builds an Sqlite database
+.Pa man.db
+to support full text searches.
+It obtains the list of directories to traverse using the
+.Fl Ar path
+and
+.Fl p
+options of
+.Xr man 1 .
+.Pp
+It supports the following options:
+.Bl -tag -width indent
+.It Fl C Ar path
+Use different
+.Xr man 1
+configuration file than the default,
+.Pa /etc/man.conf .
+.It Fl f
+Force rebuilding the index from scratch, pruning the existing one.
+.It Fl l
+Limit the parsing to only the NAME section of the pages.
+This option can be used to mimic the behavior of the classic
+.Xr apropos 1
+and also to substantially save disk space.
+.It Fl o
+Use this option to optimize the index for speed and also
+to significantly reduce disk space usage.
+This is a somewhat expensive operation.
+.It Fl q
+Print only error messages and no status updates.
+.It Fl v
+Enable verbose output.
+This prints the name of every file being parsed.
+.El
+.Pp
+As the database file is stored under
+.Pa /var/db ,
+root privileges are required to run
+.Nm .
+.Ss DATABASE SCHEMA
+The name of the FTS table is mandb and its schema is as follows:
+.Bl -column -offset indent "Column Name" "Column Description"
+.It Li section Ta The section number of the page
+.It Li name Ta The name of the page from the NAME section.
+.It Li name_desc Ta The one line description from the NAME section.
+.It Li desc Ta The DESCRIPTION section.
+.It Li lib Ta The LIBRARY section.
+.It Li return_vals Ta The RETURN VALUES section.
+.It Li env Ta The ENVIRONMENT section.
+.It Li files Ta The FILES section.
+.It Li exit_status Ta The EXIT STATUS section.
+.It Li errors Ta The ERRORS section.
+.El
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr man 1 ,
+.Xr man.conf 5
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/makemandb.c	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,1940 @@
+/*	$NetBSD	*/
+/*
+ * Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+ * Copyright (c) 2011 Kristaps Dzonsons <kristaps@bsd.lv>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD: makemandb.c,v 1.1 2012/02/07 19:13:32 joerg Exp $");
+
+#include <sys/stat.h>
+#include <sys/types.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <err.h>
+#include <md5.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+#include "apropos-utils.h"
+#include "man.h"
+#include "mandoc.h"
+#include "mdoc.h"
+#include "sqlite3.h"
+
+#define BUFLEN 1024
+#define MDOC 0	//If the page is of mdoc(7) type
+#define MAN 1	//If the page  is of man(7) type
+
+/*
+ * A data structure for holding section specific data.
+ */
+typedef struct secbuff {
+	char *data;
+	size_t buflen;	//Total length of buffer allocated initially
+	size_t offset;	// Current offset in the buffer.
+} secbuff;
+
+typedef struct makemandb_flags {
+	int optimize;
+	int limit;	// limit the indexing to only NAME section
+	int recreate;	// Database was created from scratch
+	int verbosity;	// 0: quiet, 1: default, 2: verbose
+} makemandb_flags;
+
+typedef struct mandb_rec {
+	/* Fields for mandb table */
+	char *name;	// for storing the name of the man page
+	char *name_desc; // for storing the one line description (.Nd)
+	secbuff desc; // for storing the DESCRIPTION section
+	secbuff lib; // for the LIBRARY section
+	secbuff return_vals; // RETURN VALUES
+	secbuff env; // ENVIRONMENT
+	secbuff files; // FILES
+	secbuff exit_status; // EXIT STATUS
+	secbuff diagnostics; // DIAGNOSTICS
+	secbuff errors; // ERRORS
+	char section[2];
+
+	int xr_found;
+
+	/* Fields for mandb_meta table */
+	char *md5_hash;
+	dev_t device;
+	ino_t inode;
+	time_t mtime;
+
+	/* Fields for mandb_links table */
+	char *machine;
+	char *links; //all the links to a page in a space separated form
+	char *file_path;
+
+	/* Non-db fields */
+	int page_type; //Indicates the type of page: mdoc or man
+} mandb_rec;
+
+static void append(secbuff *sbuff, const char *src);
+static void init_secbuffs(mandb_rec *);
+static void free_secbuffs(mandb_rec *);
+static int check_md5(const char *, sqlite3 *, const char *, char **);
+static void cleanup(mandb_rec *);
+static void set_section(const struct mdoc *, const struct man *, mandb_rec *);
+static void set_machine(const struct mdoc *, mandb_rec *);
+static int insert_into_db(sqlite3 *, mandb_rec *);
+static	void begin_parse(const char *, struct mparse *, mandb_rec *);
+static void pmdoc_node(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nm(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Nd(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Sh(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Xr(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_Pp(const struct mdoc_node *, mandb_rec *);
+static void pmdoc_macro_handler(const struct mdoc_node *, mandb_rec *, 
+				enum mdoct);
+static void pman_node(const struct man_node *n, mandb_rec *);
+static void pman_parse_node(const struct man_node *, secbuff *);
+static void pman_parse_name(const struct man_node *, mandb_rec *);
+static void pman_sh(const struct man_node *, mandb_rec *);
+static void pman_block(const struct man_node *, mandb_rec *);
+static void traversedir(const char *, sqlite3 *, struct mparse *);
+static void mdoc_parse_section(enum mdoc_sec, const char *, mandb_rec *);
+static void man_parse_section(enum man_sec, const struct man_node *, mandb_rec *);
+static void build_file_cache(sqlite3 *, const char *, struct stat *);
+static void update_db(sqlite3 *, struct mparse *, mandb_rec *);
+__dead static void usage(void);
+static void optimize(sqlite3 *);
+static char *parse_escape(const char *);
+static makemandb_flags mflags = { .verbosity = 1 };
+
+typedef	void (*pman_nf)(const struct man_node *n, mandb_rec *);
+typedef	void (*pmdoc_nf)(const struct mdoc_node *n, mandb_rec *);
+static	const pmdoc_nf mdocs[MDOC_MAX] = {
+	NULL, /* Ap */
+	NULL, /* Dd */
+	NULL, /* Dt */
+	NULL, /* Os */
+	pmdoc_Sh, /* Sh */
+	NULL, /* Ss */
+	pmdoc_Pp, /* Pp */
+	NULL, /* D1 */
+	NULL, /* Dl */
+	NULL, /* Bd */
+	NULL, /* Ed */
+	NULL, /* Bl */
+	NULL, /* El */
+	NULL, /* It */
+	NULL, /* Ad */
+	NULL, /* An */
+	NULL, /* Ar */
+	NULL, /* Cd */
+	NULL, /* Cm */
+	NULL, /* Dv */
+	NULL, /* Er */
+	NULL, /* Ev */
+	NULL, /* Ex */
+	NULL, /* Fa */
+	NULL, /* Fd */
+	NULL, /* Fl */
+	NULL, /* Fn */
+	NULL, /* Ft */
+	NULL, /* Ic */
+	NULL, /* In */
+	NULL, /* Li */
+	pmdoc_Nd, /* Nd */
+	pmdoc_Nm, /* Nm */
+	NULL, /* Op */
+	NULL, /* Ot */
+	NULL, /* Pa */
+	NULL, /* Rv */
+	NULL, /* St */
+	NULL, /* Va */
+	NULL, /* Vt */
+	pmdoc_Xr, /* Xr */
+	NULL, /* %A */
+	NULL, /* %B */
+	NULL, /* %D */
+	NULL, /* %I */
+	NULL, /* %J */
+	NULL, /* %N */
+	NULL, /* %O */
+	NULL, /* %P */
+	NULL, /* %R */
+	NULL, /* %T */
+	NULL, /* %V */
+	NULL, /* Ac */
+	NULL, /* Ao */
+	NULL, /* Aq */
+	NULL, /* At */
+	NULL, /* Bc */
+	NULL, /* Bf */
+	NULL, /* Bo */
+	NULL, /* Bq */
+	NULL, /* Bsx */
+	NULL, /* Bx */
+	NULL, /* Db */
+	NULL, /* Dc */
+	NULL, /* Do */
+	NULL, /* Dq */
+	NULL, /* Ec */
+	NULL, /* Ef */
+	NULL, /* Em */
+	NULL, /* Eo */
+	NULL, /* Fx */
+	NULL, /* Ms */
+	NULL, /* No */
+	NULL, /* Ns */
+	NULL, /* Nx */
+	NULL, /* Ox */
+	NULL, /* Pc */
+	NULL, /* Pf */
+	NULL, /* Po */
+	NULL, /* Pq */
+	NULL, /* Qc */
+	NULL, /* Ql */
+	NULL, /* Qo */
+	NULL, /* Qq */
+	NULL, /* Re */
+	NULL, /* Rs */
+	NULL, /* Sc */
+	NULL, /* So */
+	NULL, /* Sq */
+	NULL, /* Sm */
+	NULL, /* Sx */
+	NULL, /* Sy */
+	NULL, /* Tn */
+	NULL, /* Ux */
+	NULL, /* Xc */
+	NULL, /* Xo */
+	NULL, /* Fo */
+	NULL, /* Fc */
+	NULL, /* Oo */
+	NULL, /* Oc */
+	NULL, /* Bk */
+	NULL, /* Ek */
+	NULL, /* Bt */
+	NULL, /* Hf */
+	NULL, /* Fr */
+	NULL, /* Ud */
+	NULL, /* Lb */
+	NULL, /* Lp */
+	NULL, /* Lk */
+	NULL, /* Mt */
+	NULL, /* Brq */
+	NULL, /* Bro */
+	NULL, /* Brc */
+	NULL, /* %C */
+	NULL, /* Es */
+	NULL, /* En */
+	NULL, /* Dx */
+	NULL, /* %Q */
+	NULL, /* br */
+	NULL, /* sp */
+	NULL, /* %U */
+	NULL, /* Ta */
+};
+
+static	const pman_nf mans[MAN_MAX] = {
+	NULL,	//br
+	NULL,	//TH
+	pman_sh, //SH
+	NULL,	//SS
+	NULL,	//TP
+	NULL,	//LP
+	NULL,	//PP
+	NULL,	//P
+	NULL,	//IP
+	NULL,	//HP
+	NULL,	//SM
+	NULL,	//SB
+	NULL,	//BI
+	NULL,	//IB
+	NULL,	//BR
+	NULL,	//RB
+	NULL,	//R
+	pman_block,	//B
+	NULL,	//I
+	NULL,	//IR
+	NULL,	//RI
+	NULL,	//na
+	NULL,	//sp
+	NULL,	//nf
+	NULL,	//fi
+	NULL,	//RE
+	NULL,	//RS
+	NULL,	//DT
+	NULL,	//UC
+	NULL,	//PD
+	NULL,	//AT
+	NULL,	//in
+	NULL,	//ft
+};
+
+
+int
+main(int argc, char *argv[])
+{
+	FILE *file;
+	const char *sqlstr, *manconf = NULL;
+	char *line, *command;
+	char *errmsg;
+	int ch;
+	struct mparse *mp;
+	sqlite3 *db;
+	ssize_t len;
+	size_t linesize;
+	struct mandb_rec rec;
+
+	while ((ch = getopt(argc, argv, "C:floqv")) != -1) {
+		switch (ch) {
+		case 'C':
+			manconf = optarg;
+			break;
+		case 'f':
+			remove(DBPATH);
+			mflags.recreate = 1;
+			break;
+		case 'l':
+			mflags.limit = 1;
+			break;
+		case 'o':
+			mflags.optimize = 1;
+			break;
+		case 'q':
+			mflags.verbosity = 0;
+			break;
+		case 'v':
+			mflags.verbosity = 2;
+			break;
+		default:
+			usage();
+		}
+	}
+
+	memset(&rec, 0, sizeof(rec));
+
+	init_secbuffs(&rec);
+	mp = mparse_alloc(MPARSE_AUTO, MANDOCLEVEL_FATAL, NULL, NULL);
+
+	if ((db = init_db(MANDB_CREATE)) == NULL)
+		exit(EXIT_FAILURE);
+
+	sqlite3_exec(db, "PRAGMA synchronous = 0", NULL, NULL, 	&errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		close_db(db);
+		exit(EXIT_FAILURE);
+	}
+
+	sqlite3_exec(db, "ATTACH DATABASE \':memory:\' AS metadb", NULL, NULL,
+	    &errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		close_db(db);
+		exit(EXIT_FAILURE);
+	}
+
+	if (manconf) {
+		char *arg;
+		size_t command_len = shquote(manconf, NULL, 0) + 1;
+		arg = malloc(command_len );
+		shquote(manconf, arg, command_len);
+		easprintf(&command, "man -p -C %s", arg);
+		free(arg);
+	} else {
+		command = estrdup("man -p");
+	}
+
+	/* Call man -p to get the list of man page dirs */
+	if ((file = popen(command, "r")) == NULL) {
+		close_db(db);
+		err(EXIT_FAILURE, "fopen failed");
+	}
+	free(command);
+
+	/* Begin the transaction for indexing the pages	*/
+	sqlite3_exec(db, "BEGIN", NULL, NULL, &errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		exit(EXIT_FAILURE);
+	}
+		
+	sqlstr = "CREATE TABLE IF NOT EXISTS metadb.file_cache(device, inode,"
+		 " mtime, file PRIMARY KEY);"
+		 "CREATE UNIQUE INDEX IF NOT EXISTS metadb.index_file_cache_dev"
+		 " ON file_cache (device, inode)";
+
+	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		close_db(db);
+		exit(EXIT_FAILURE);
+	}
+
+	if (mflags.verbosity)
+		printf("Building temporary file cache\n");
+	line = NULL;
+	linesize = 0;
+	while ((len = getline(&line, &linesize, file)) != -1) {
+		/* Replace the new line character at the end of string with '\0' */
+		line[len - 1] = '\0';
+		/* Traverse the man page directories and parse the pages */
+		traversedir(line, db, mp);
+	}
+	free(line);
+
+	if (pclose(file) == -1) {
+		close_db(db);
+		cleanup(&rec);
+		free_secbuffs(&rec);
+		err(EXIT_FAILURE, "pclose error");
+	}
+
+	update_db(db, mp, &rec);
+	mparse_free(mp);
+	free_secbuffs(&rec);
+
+	/* Commit the transaction */
+	sqlite3_exec(db, "COMMIT", NULL, NULL, &errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		exit(EXIT_FAILURE);
+	}
+
+	if (mflags.optimize)
+		optimize(db);
+
+	close_db(db);
+	return 0;
+}
+
+/*
+ * traversedir --
+ *  Traverses the given directory recursively and passes all the man page files
+ *  in the way to build_file_cache()
+ */
+static void
+traversedir(const char *file, sqlite3 *db, struct mparse *mp)
+{
+	struct stat sb;
+	struct dirent *dirp;
+	DIR *dp;
+	char *buf;
+
+	if (stat(file, &sb) < 0) {
+		warn("stat failed: %s", file);
+		return;
+	}
+	
+	/* If it is a regular file or a symlink, pass it to build_cache() */
+	if (S_ISREG(sb.st_mode) || S_ISLNK(sb.st_mode)) {
+		build_file_cache(db, file, &sb);
+		return;
+	}
+	
+	/* If it is a directory, traverse it recursively */
+	if (S_ISDIR(sb.st_mode)) {
+		if ((dp = opendir(file)) == NULL) {
+			warn("opendir error: %s", file);
+			return;
+		}
+		
+		while ((dirp = readdir(dp)) != NULL) {
+			/* Avoid . and .. entries in a directory */
+			if (strncmp(dirp->d_name, ".", 1)) {
+				easprintf(&buf, "%s/%s", file, dirp->d_name);
+				traversedir(buf, db, mp);
+				free(buf);
+			}
+		}
+		closedir(dp);
+	}
+}
+
+/* build_file_cache --
+ *   This function generates an md5 hash of the file passed as it's 2nd parameter
+ *   and stores it in a temporary table file_cache along with the full file path.
+ *   This is done to support incremental updation of the database.
+ *   The temporary table file_cache is dropped thereafter in the function
+ *   update_db(), once the database has been updated.
+ */
+static void
+build_file_cache(sqlite3 *db, const char *file, struct stat *sb)
+{
+	const char *sqlstr;
+	sqlite3_stmt *stmt = NULL;
+	int rc, idx;
+	assert(file != NULL);
+	dev_t device_cache = sb->st_dev;
+	ino_t inode_cache = sb->st_ino;
+	time_t mtime_cache = sb->st_mtime;
+
+	sqlstr = "INSERT INTO metadb.file_cache VALUES (:device, :inode,"
+		 " :mtime, :file)";
+	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		return;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":device");
+	rc = sqlite3_bind_int64(stmt, idx, device_cache);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		return;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":inode");
+	rc = sqlite3_bind_int64(stmt, idx, inode_cache);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		return;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+	rc = sqlite3_bind_int64(stmt, idx, mtime_cache);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		return;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":file");
+	rc = sqlite3_bind_text(stmt, idx, file, -1, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		return;
+	}
+
+	sqlite3_step(stmt);
+	sqlite3_finalize(stmt);
+}
+
+static void
+update_existing_entry(sqlite3 *db, const char *file, const char *hash,
+    mandb_rec *rec, int *new_count, int *link_count, int *err_count)
+{
+	int update_count, rc, idx;
+	const char *inner_sqlstr;
+	sqlite3_stmt *inner_stmt;
+
+	update_count = sqlite3_total_changes(db);
+	inner_sqlstr = "UPDATE mandb_meta SET device = :device,"
+		       " inode = :inode, mtime = :mtime WHERE"
+		       " md5_hash = :md5 AND file = :file AND"
+		       " (device <> :device2 OR inode <> "
+		       "  :inode2 OR mtime <> :mtime2)";
+	rc = sqlite3_prepare_v2(db, inner_sqlstr, -1, &inner_stmt, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		return;
+	}
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":device");
+	sqlite3_bind_int64(inner_stmt, idx, rec->device);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode");
+	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime");
+	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":md5");
+	sqlite3_bind_text(inner_stmt, idx, hash, -1, NULL);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":file");
+	sqlite3_bind_text(inner_stmt, idx, file, -1, NULL);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":device2");
+	sqlite3_bind_int64(inner_stmt, idx, rec->device);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":inode2");
+	sqlite3_bind_int64(inner_stmt, idx, rec->inode);
+	idx = sqlite3_bind_parameter_index(inner_stmt, ":mtime2");
+	sqlite3_bind_int64(inner_stmt, idx, rec->mtime);
+
+	rc = sqlite3_step(inner_stmt);
+	if (rc == SQLITE_DONE) {
+		/* Check if an update has been performed. */
+		if (update_count != sqlite3_total_changes(db)) {
+			if (mflags.verbosity)
+				printf("Updated %s\n", file);
+			(*new_count)++;
+		} else {
+			/* Otherwise it was a hardlink. */
+			(*link_count)++;
+		}
+	} else {
+		warnx("Could not update the meta data for %s", file);
+		(*err_count)++;
+	}
+	sqlite3_finalize(inner_stmt);
+}
+
+/* update_db --
+ *	Does an incremental updation of the database by checking the file_cache.
+ *	It parses and adds the pages which are present in file_cache,
+ *	but not in the database.
+ *	It also removes the pages which are present in the databse,
+ *	but not in the file_cache.
+ */
+static void
+update_db(sqlite3 *db, struct mparse *mp, mandb_rec *rec)
+{
+	const char *sqlstr;
+	sqlite3_stmt *stmt = NULL;
+	const char *file;
+	char *errmsg = NULL;
+	char *buf = NULL;
+	int new_count = 0;	/* Counter for newly indexed/updated pages */
+	int total_count = 0;	/* Counter for total number of pages */
+	int err_count = 0;	/* Counter for number of failed pages */
+	int link_count = 0;	/* Counter for number of hard/sym links */
+	int md5_status;
+	int rc;
+
+	sqlstr = "SELECT device, inode, mtime, file FROM metadb.file_cache"
+		 " EXCEPT SELECT device, inode, mtime, file from mandb_meta";
+
+	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		close_db(db);
+		errx(EXIT_FAILURE, "Could not query file cache");
+	}
+
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		total_count++;
+		rec->device = sqlite3_column_int64(stmt, 0);
+		rec->inode = sqlite3_column_int64(stmt, 1);
+		rec->mtime = sqlite3_column_int64(stmt, 2);
+		file = (const char *) sqlite3_column_text(stmt, 3);
+		md5_status = check_md5(file, db, "mandb_meta", &buf);
+		assert(buf != NULL);
+		if (md5_status == -1) {
+			warnx("An error occurred in checking md5 value"
+			      " for file %s", file);
+			err_count++;
+			continue;
+		}
+
+		if (md5_status == 0) {
+			/*
+			 * The MD5 hash is already present in the database,
+			 * so simply update the metadata, ignoring symlinks.
+			 */
+			struct stat sb;
+			stat(file, &sb);
+			if (S_ISLNK(sb.st_mode)) {
+				free(buf);
+				link_count++;
+				continue;
+			}
+			update_existing_entry(db, file, buf, rec,
+			    &new_count, &link_count, &err_count);
+			free(buf);
+			continue;
+		}
+
+		if (md5_status == 1) {
+			/*
+			 * The MD5 hash was not present in the database.
+			 * This means is either a new file or an updated file.
+			 * We should go ahead with parsing.
+			 */
+			if (mflags.verbosity > 1)
+				printf("Parsing: %s\n", file);
+			rec->md5_hash = buf;
+			rec->file_path = estrdup(file);
+			// file_path is freed by insert_into_db itself.
+			begin_parse(file, mp, rec);
+			if (insert_into_db(db, rec) < 0) {
+				warnx("Error in indexing %s", file);
+				err_count++;
+			} else {
+				new_count++;
+			}
+		}
+	}
+	
+	sqlite3_finalize(stmt);
+	
+	if (mflags.verbosity) {
+		printf("Total Number of new or updated pages enountered = %d\n"
+			"Total number of pages that were successfully"
+			" indexed/updated = %d\n"
+			"Total number of (hard or symbolic) links found = %d\n"
+			"Total number of pages that could not be indexed"
+			" due to errors = %d\n",
+			total_count, new_count, link_count, err_count);
+	}
+
+	if (mflags.recreate == 0)
+		return;
+
+	if (mflags.verbosity)
+		printf("Deleting stale index entries\n");
+
+	sqlstr = "DELETE FROM mandb_meta WHERE file NOT IN"
+		 " (SELECT file FROM metadb.file_cache);"
+		 "DROP TABLE metadb.file_cache;"
+		 "DELETE FROM mandb WHERE rowid NOT IN"
+		 " (SELECT id FROM mandb_meta);";
+
+	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+	if (errmsg != NULL) {
+		warnx("Removing old entries failed: %s", errmsg);
+		warnx("Please rebuild database from scratch with -f.");
+		free(errmsg);
+		return;
+	}
+}
+
+/*
+ * begin_parse --
+ *  parses the man page using libmandoc
+ */
+static void
+begin_parse(const char *file, struct mparse *mp, mandb_rec *rec)
+{
+	struct mdoc *mdoc;
+	struct man *man;
+	mparse_reset(mp);
+
+	rec->xr_found = 0;
+
+	if (mparse_readfd(mp, -1, file) >= MANDOCLEVEL_FATAL) {
+		warnx("%s: Parse failure", file);
+		return;
+	}
+
+	mparse_result(mp, &mdoc, &man);
+	if (mdoc == NULL && man == NULL) {
+		warnx("Not a man(7) or mdoc(7) page");
+		return;
+	}
+
+	set_machine(mdoc, rec);
+	set_section(mdoc, man, rec);
+	if (mdoc) {
+		rec->page_type = MDOC;
+		pmdoc_node(mdoc_node(mdoc), rec);
+	} else {
+		rec->page_type = MAN;
+		pman_node(man_node(man), rec);
+	}
+}
+
+/*
+ * set_section --
+ *  Extracts the section number and normalizes it to only the numeric part
+ *  (Which should be the first character of the string).
+ */
+static void
+set_section(const struct mdoc *md, const struct man *m, mandb_rec *rec)
+{
+	if (md) {
+		const struct mdoc_meta *md_meta = mdoc_meta(md);
+		rec->section[0] = md_meta->msec[0];
+	} else if (m) {
+		const struct man_meta *m_meta = man_meta(m);
+		rec->section[0] = m_meta->msec[0];
+	}
+}
+
+/*
+ * get_machine --
+ *  Extracts the machine architecture information if available.
+ */
+static void
+set_machine(const struct mdoc *md, mandb_rec *rec)
+{
+	if (md == NULL)
+		return;
+	const struct mdoc_meta *md_meta = mdoc_meta(md);
+	if (md_meta->arch)
+		rec->machine = estrdup(md_meta->arch);
+}
+
+static void
+pmdoc_node(const struct mdoc_node *n, mandb_rec *rec)
+{
+
+	if (n == NULL)
+		return;
+
+	switch (n->type) {
+	case (MDOC_BODY):
+		/* FALLTHROUGH */
+	case (MDOC_TAIL):
+		/* FALLTHROUGH */
+	case (MDOC_ELEM):
+		if (mdocs[n->tok] == NULL)
+			break;
+		(*mdocs[n->tok])(n, rec);
+		break;
+	default:
+		break;
+	}
+
+	pmdoc_node(n->child, rec);
+	pmdoc_node(n->next, rec);
+}
+
+/*
+ * pmdoc_Nm --
+ *  Extracts the Name of the manual page from the .Nm macro
+ */
+static void
+pmdoc_Nm(const struct mdoc_node *n, mandb_rec *rec)
+{
+	if (n->sec != SEC_NAME)
+		return;
+
+	for (n = n->child; n; n = n->next) {
+		if (n->type == MDOC_TEXT) {
+			concat(&rec->name, n->string);
+		}
+	}
+}
+
+/*
+ * pmdoc_Nd --
+ *  Extracts the one line description of the man page from the .Nd macro
+ */
+static void
+pmdoc_Nd(const struct mdoc_node *n, mandb_rec *rec)
+{
+	/*
+	 * A static variable for keeping track of whether a Xr macro was seen
+	 * previously.
+	 */
+	char *buf = NULL;
+	char *temp;
+
+	if (n == NULL)
+		return;
+
+	if (n->type == MDOC_TEXT) {
+		if (rec->xr_found && n->next) {
+			/*
+			 * An Xr macro was seen previously, so parse this
+			 * and the next node.
+			 */
+			temp = estrdup(n->string);
+			n = n->next;
+			easprintf(&buf, "%s(%s)", temp, n->string);
+			concat(&rec->name_desc, buf);
+			free(buf);
+			free(temp);
+		} else {
+			concat(&rec->name_desc, n->string);
+		}
+		rec->xr_found = 0;
+	} else if (mdocs[n->tok] == pmdoc_Xr) {
+		/* Remember that we have encountered an Xr macro */
+		rec->xr_found = 1;
+	}
+
+	if (n->child)
+		pmdoc_Nd(n->child, rec);
+
+	if(n->next)
+		pmdoc_Nd(n->next, rec);
+}
+
+/*
+ * pmdoc_macro_handler--
+ *  This function is a single point of handling all the special macros that we
+ *  want to handle especially. For example the .Xr macro for properly parsing
+ *  the referenced page name along with the section number, or the .Pp macro
+ *  for adding a new line whenever we encounter it.
+ */
+static void
+pmdoc_macro_handler(const struct mdoc_node *n, mandb_rec *rec, enum mdoct doct)
+{
+	const struct mdoc_node *sn;
+	assert(n);
+
+	switch (doct) {
+	/*  Parse the man page references.
+	 * Basically the .Xr macros are used like:
+	 *  .Xr ls 1
+ 	 *  and formatted like this:
+	 *  ls(1)
+	 *  Prepare a buffer to format the data like the above example and call
+	 *  pmdoc_parse_section to append it.
+	 */
+	case MDOC_Xr:
+		n = n->child;
+		while (n->type != MDOC_TEXT && n->next)
+			n = n->next;
+
+		if (n && n->type != MDOC_TEXT)
+			return;
+		sn = n;
+		if (n->next)
+			n = n->next;
+
+		while (n->type != MDOC_TEXT && n->next)
+			n = n->next;
+
+		if (n && n->type == MDOC_TEXT) {
+			size_t len = strlen(sn->string);
+			char *buf = emalloc(len + 4);
+			memcpy(buf, sn->string, len);
+			buf[len] = '(';
+			buf[len + 1] = n->string[0];
+			buf[len + 2] = ')';
+			buf[len + 3] = 0;
+			mdoc_parse_section(n->sec, buf, rec);
+			free(buf);
+		}
+
+		break;
+
+	/* Parse the .Pp macro to add a new line */
+	case MDOC_Pp:
+		if (n->type == MDOC_TEXT)
+			mdoc_parse_section(n->sec, "\n", rec);
+		break;
+	default:
+		break;
+	}
+
+}
+
+/*
+ * pmdoc_Xr, pmdoc_Pp--
+ *  Empty stubs.
+ *  The parser calls these functions each time it encounters
+ *  a .Xr or .Pp macro. We are parsing all the data from
+ *  the pmdoc_Sh function, so don't do anything here.
+ *  (See if else blocks in pmdoc_Sh.)
+ */
+static void
+pmdoc_Xr(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+static void
+pmdoc_Pp(const struct mdoc_node *n, mandb_rec *rec)
+{
+}
+
+/*
+ * pmdoc_Sh --
+ *  Called when a .Sh macro is encountered and loops through its body, calling
+ *  mdoc_parse_section to append the data to the section specific buffer.
+ *  Two special macros which may occur inside the body of Sh are .Nm and .Xr and
+ *  they need special handling, thus the separate if branches for them.
+ */
+static void
+pmdoc_Sh(const struct mdoc_node *n, mandb_rec *rec)
+{
+	if (n == NULL)
+		return;
+	int xr_found = 0;
+
+	if (n->type == MDOC_TEXT) {
+		mdoc_parse_section(n->sec, n->string, rec);
+	} else if (mdocs[n->tok] == pmdoc_Nm && rec->name != NULL) {
+		/*
+		 * When encountering a .Nm macro, substitute it
+		 * with its previously cached value of the argument.
+		 */
+		mdoc_parse_section(n->sec, rec->name, rec);
+	} else if (mdocs[n->tok] == pmdoc_Xr) {
+		/*
+		 * When encountering other inline macros,
+		 * call pmdoc_macro_handler.
+		 */
+		pmdoc_macro_handler(n, rec, MDOC_Xr);
+		xr_found = 1;
+	} else if (mdocs[n->tok] == pmdoc_Pp) {
+		pmdoc_macro_handler(n, rec, MDOC_Pp);
+	}
+
+	/*
+	 * If an Xr macro was encountered then the child node has
+	 * already been explored by pmdoc_macro_handler.
+	 */
+	if (xr_found == 0)
+		pmdoc_Sh(n->child, rec);
+	pmdoc_Sh(n->next, rec);
+}
+
+/*
+ * mdoc_parse_section--
+ *  Utility function for parsing sections of the mdoc type pages.
+ *  Takes two params:
+ *   1. sec is an enum which indicates the section in which we are present
+ *   2. string is the string which we need to append to the secbuff for this
+ *      particular section.
+ *  The function appends string to the global section buffer and returns.
+ */
+static void
+mdoc_parse_section(enum mdoc_sec sec, const char *string, mandb_rec *rec)
+{
+	/*
+	 * If the user specified the 'l' flag, then parse and store only the
+	 * NAME section. Ignore the rest.
+	 */
+	if (mflags.limit)
+		return;
+
+	switch (sec) {
+	case SEC_LIBRARY:
+		append(&rec->lib, string);
+		break;
+	case SEC_RETURN_VALUES:
+		append(&rec->return_vals, string);
+		break;
+	case SEC_ENVIRONMENT:
+		append(&rec->env, string);
+		break;
+	case SEC_FILES:
+		append(&rec->files, string);
+		break;
+	case SEC_EXIT_STATUS:
+		append(&rec->exit_status, string);
+		break;
+	case SEC_DIAGNOSTICS:
+		append(&rec->diagnostics, string);
+		break;
+	case SEC_ERRORS:
+		append(&rec->errors, string);
+		break;
+	case SEC_NAME:
+	case SEC_SYNOPSIS:
+	case SEC_EXAMPLES:
+	case SEC_STANDARDS:
+	case SEC_HISTORY:
+	case SEC_AUTHORS:
+	case SEC_BUGS:
+		break;
+	default:
+		append(&rec->desc, string);
+		break;
+	}
+}
+
+static void
+pman_node(const struct man_node *n, mandb_rec *rec)
+{
+	if (n == NULL)
+		return;
+
+	switch (n->type) {
+	case (MAN_BODY):
+		/* FALLTHROUGH */
+	case (MAN_TAIL):
+		/* FALLTHROUGH */
+	case (MAN_BLOCK):
+		/* FALLTHROUGH */
+	case (MAN_ELEM):
+		if (mans[n->tok] != NULL)
+			(*mans[n->tok])(n, rec);
+		break;
+	default:
+		break;
+	}
+
+	pman_node(n->child, rec);
+	pman_node(n->next, rec);
+}
+
+/* 
+ * pman_parse_name --
+ *  Parses the NAME section and puts the complete content in the name_desc
+ *  variable.
+ */
+static void
+pman_parse_name(const struct man_node *n, mandb_rec *rec)
+{
+	if (n == NULL)
+		return;
+
+	if (n->type == MAN_TEXT) {
+		char *tmp = parse_escape(n->string);
+		concat(&rec->name_desc, tmp);
+		free(tmp);
+	}
+
+	if (n->child)
+		pman_parse_name(n->child, rec);
+
+	if(n->next)
+		pman_parse_name(n->next, rec);
+}
+
+/*
+ * A stub function to be able to parse the macros like .B embedded inside
+ * a section.
+ */
+static void
+pman_block(const struct man_node *n, mandb_rec *rec)
+{
+}
+
+/* 
+ * pman_sh --
+ * This function does one of the two things:
+ *  1. If the present section is NAME, then it will:
+ *    (a) Extract the name of the page (in case of multiple comma separated
+ *        names, it will pick up the first one).
+ *    (b) Build a space spearated list of all the symlinks/hardlinks to
+ *        this page and store in the buffer 'links'. These are extracted from
+ *        the comma separated list of names in the NAME section as well.
+ *    (c) Move on to the one line description section, which is after the list
+ *        of names in the NAME section.
+ *  2. Otherwise, it will check the section name and call the man_parse_section
+ *     function, passing the enum corresponding that section.
+ */
+static void
+pman_sh(const struct man_node *n, mandb_rec *rec)
+{
+	static const struct {
+		enum man_sec section;
+		const char *header;
+	} mapping[] = {
+	    { MANSEC_DESCRIPTION, "DESCRIPTION" },
+	    { MANSEC_SYNOPSIS, "SYNOPSIS" },
+	    { MANSEC_LIBRARY, "LIBRARY" },
+	    { MANSEC_ERRORS, "ERRORS" },
+	    { MANSEC_FILES, "FILES" },
+	    { MANSEC_RETURN_VALUES, "RETURN VALUE" },
+	    { MANSEC_RETURN_VALUES, "RETURN VALUES" },
+	    { MANSEC_EXIT_STATUS, "EXIT STATUS" },
+	    { MANSEC_EXAMPLES, "EXAMPLES" },
+	    { MANSEC_EXAMPLES, "EXAMPLE" },
+	    { MANSEC_STANDARDS, "STANDARDS" },
+	    { MANSEC_HISTORY, "HISTORY" },
+	    { MANSEC_BUGS, "BUGS" },
+	    { MANSEC_AUTHORS, "AUTHORS" },
+	    { MANSEC_COPYRIGHT, "COPYRIGHT" },
+	};
+	const struct man_node *head;
+	char *name_desc;
+	int sz;
+	size_t i;
+
+	if ((head = n->parent->head) == NULL || (head = head->child) == NULL ||
+	    head->type != MAN_TEXT)
+		return;
+
+	/*
+	 * Check if this section should be extracted and
+	 * where it should be stored. Handled the trival cases first.
+	 */
+	for (i = 0; i < sizeof(mapping) / sizeof(mapping[0]); ++i) {
+		if (strcmp(head->string, mapping[i].header) == 0) {
+			man_parse_section(mapping[i].section, n, rec);
+			return;
+		}
+	}
+
+	if (strcmp(head->string, "NAME") == 0) {
+		/*
+		 * We are in the NAME section.
+		 * pman_parse_name will put the complete content in name_desc.
+		 */
+		pman_parse_name(n, rec);
+
+		name_desc = rec->name_desc;
+
+		/* Remove any leading spaces. */
+		while (name_desc[0] == ' ')
+			name_desc++;
+			
+		/* If the line begins with a "\&", avoid those */
+		if (name_desc[0] == '\\' && name_desc[1] == '&')
+			name_desc += 2;
+
+		/* Now name_desc should be left with a comma-space
+		 * separated list of names and the one line description
+		 * of the page:
+		 *     "a, b, c \- sample description"
+		 * Take out the first name, before the first comma
+		 * (or space) and store it in rec->name.
+		 * If the page has aliases then they should be
+		 * in the form of a comma separated list.
+		 * Keep looping while there is a comma in name_desc,
+		 * extract the alias name and store in rec->links.
+		 * When there are no more commas left, break out.
+		 */
+		int has_alias = 0;	// Any more aliases left?
+		while (*name_desc) {
+			/* Remove any leading spaces. */
+			if (name_desc[0] == ' ') {
+				name_desc++;
+				continue;
+			}
+			sz = strcspn(name_desc, ", ");
+
+			/* Extract the first term and store it in rec->name. */
+			if (rec->name == NULL) {
+				if (name_desc[sz] == ',')
+					has_alias = 1;
+				name_desc[sz] = 0;
+				rec->name = emalloc(sz + 1);
+				memcpy(rec->name, name_desc, sz + 1);
+				name_desc += sz + 1;
+				continue;
+			}
+
+			/*
+			 * Once rec->name is set, rest of the names
+			 * are to be treated as links or aliases.
+			 */
+			if (rec->name && has_alias) {
+				if (name_desc[sz] != ',') {
+					/* No more commas left -->
+					 * no more aliases to take out
+					 */
+					has_alias = 0;
+				}
+				name_desc[sz] = 0;
+				concat2(&rec->links, name_desc, sz);
+				name_desc += sz + 1;
+				continue;
+			}
+			break;
+		}
+
+		/* Parse any escape sequences that might be there */
+		char *temp = parse_escape(name_desc);
+		free(rec->name_desc);
+		rec->name_desc = temp;
+		temp = parse_escape(rec->name);
+		free(rec->name);
+		rec->name = temp;
+		return;
+	}
+
+	/* The RETURN VALUE section might be specified in multiple ways */
+	if (strcmp(head->string, "RETURN") == 0 &&
+	    head->next != NULL && head->next->type == MAN_TEXT &&
+	    (strcmp(head->next->string, "VALUE") == 0 ||
+	    strcmp(head->next->string, "VALUES") == 0)) {
+		man_parse_section(MANSEC_RETURN_VALUES, n, rec);
+		return;
+	}
+
+	/*
+	 * EXIT STATUS section can also be specified all on one line or on two
+	 * separate lines.
+	 */
+	if (strcmp(head->string, "EXIT") == 0 &&
+	    head->next != NULL && head->next->type == MAN_TEXT &&
+	    strcmp(head->next->string, "STATUS") == 0) {
+		man_parse_section(MANSEC_EXIT_STATUS, n, rec);
+		return;
+	}
+
+	/* Store the rest of the content in desc. */
+	man_parse_section(MANSEC_NONE, n, rec);
+}
+
+/*
+ * pman_parse_node --
+ *  Generic function to iterate through a node. Usually called from 
+ *  man_parse_section to parse a particular section of the man page.
+ */
+static void
+pman_parse_node(const struct man_node *n, secbuff *s)
+{
+	if (n == NULL)
+		return;
+
+	if (n->type == MAN_TEXT)
+		append(s, n->string);
+		
+	pman_parse_node(n->child, s);
+	pman_parse_node(n->next, s);
+}
+
+/*
+ * man_parse_section --
+ *  Takes two parameters: 
+ *   sec: Tells which section we are present in
+ *   n: Is the present node of the AST.
+ * Depending on the section, we call pman_parse_node to parse that section and
+ * concatenate the content from that section into the buffer for that section.
+ */
+static void
+man_parse_section(enum man_sec sec, const struct man_node *n, mandb_rec *rec)
+{
+	/*
+	 * If the user sepecified the 'l' flag then just parse
+	 * the NAME section, ignore the rest.
+	 */
+	if (mflags.limit)
+		return;
+
+	switch (sec) {
+	case MANSEC_LIBRARY:
+		pman_parse_node(n, &rec->lib);
+		break;
+	case MANSEC_RETURN_VALUES:
+		pman_parse_node(n, &rec->return_vals);
+		break;
+	case MANSEC_ENVIRONMENT:
+		pman_parse_node(n, &rec->env);
+		break;
+	case MANSEC_FILES:
+		pman_parse_node(n, &rec->files);
+		break;
+	case MANSEC_EXIT_STATUS:
+		pman_parse_node(n, &rec->exit_status);
+		break;
+	case MANSEC_DIAGNOSTICS:
+		pman_parse_node(n, &rec->diagnostics);
+		break;
+	case MANSEC_ERRORS:
+		pman_parse_node(n, &rec->errors);
+		break;
+	case MANSEC_NAME:
+	case MANSEC_SYNOPSIS:
+	case MANSEC_EXAMPLES:
+	case MANSEC_STANDARDS:
+	case MANSEC_HISTORY:
+	case MANSEC_BUGS:
+	case MANSEC_AUTHORS:
+	case MANSEC_COPYRIGHT:
+		break;
+	default:
+		pman_parse_node(n, &rec->desc);
+		break;
+	}
+
+}
+
+/*
+ * insert_into_db --
+ *  Inserts the parsed data of the man page in the Sqlite databse.
+ *  If any of the values is NULL, then we cleanup and return -1 indicating
+ *  an error.
+ *  Otherwise, store the data in the database and return 0.
+ */
+static int
+insert_into_db(sqlite3 *db, mandb_rec *rec)
+{
+	int rc = 0;
+	int idx = -1;
+	const char *sqlstr = NULL;
+	sqlite3_stmt *stmt = NULL;
+	char *ln = NULL;
+	char *errmsg = NULL;
+	long int mandb_rowid;
+	
+	/*
+	 * At the very minimum we want to make sure that we store
+	 * the following data:
+	 *   Name, one line description, and the MD5 hash
+	 */		
+	if (rec->name == NULL || rec->name_desc == NULL ||
+	    rec->md5_hash == NULL) {
+		cleanup(rec);
+		return -1;
+	}
+
+	/* Write null byte at the end of all the sec_buffs */
+	rec->desc.data[rec->desc.offset] = 0;
+	rec->lib.data[rec->lib.offset] = 0;
+	rec->env.data[rec->env.offset] = 0;
+	rec->return_vals.data[rec->return_vals.offset] = 0;
+	rec->exit_status.data[rec->exit_status.offset] = 0;
+	rec->files.data[rec->files.offset] = 0;
+	rec->diagnostics.data[rec->diagnostics.offset] = 0;
+	rec->errors.data[rec->errors.offset] = 0;
+
+	/*
+	 * In case of a mdoc page: (sorry, no better place to put this code)
+	 * parse the comma separated list of names of man pages,
+	 * the first name will be stored in the mandb table, rest will be
+	 * treated as links and put in the mandb_links table.
+	 */
+	if (rec->page_type == MDOC) {
+		char *tmp;
+		rec->links = estrdup(rec->name);
+		free(rec->name);
+		int sz = strcspn(rec->links, " \0");
+		rec->name = emalloc(sz + 1);
+		memcpy(rec->name, rec->links, sz);
+		if(rec->name[sz - 1] == ',')
+			rec->name[sz - 1] = 0;
+		else
+			rec->name[sz] = 0;
+		while (rec->links[sz] == ' ')
+			++sz;
+		tmp = estrdup(rec->links + sz);
+		free(rec->links);
+		rec->links = tmp;
+	}
+
+/*------------------------ Populate the mandb table---------------------------*/
+	sqlstr = "INSERT INTO mandb VALUES (:section, :name, :name_desc, :desc,"
+		 " :lib, :return_vals, :env, :files, :exit_status,"
+		 " :diagnostics, :errors, :md5_hash, :machine)";
+
+	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+	if (rc != SQLITE_OK)
+		goto Out;
+
+	idx = sqlite3_bind_parameter_index(stmt, ":name");
+	rc = sqlite3_bind_text(stmt, idx, rec->name, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":section");
+	rc = sqlite3_bind_text(stmt, idx, rec->section, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":name_desc");
+	rc = sqlite3_bind_text(stmt, idx, rec->name_desc, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":desc");
+	rc = sqlite3_bind_text(stmt, idx, rec->desc.data,
+	                       rec->desc.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":lib");
+	rc = sqlite3_bind_text(stmt, idx, rec->lib.data, rec->lib.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":return_vals");
+	rc = sqlite3_bind_text(stmt, idx, rec->return_vals.data,
+	                      rec->return_vals.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":env");
+	rc = sqlite3_bind_text(stmt, idx, rec->env.data, rec->env.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":files");
+	rc = sqlite3_bind_text(stmt, idx, rec->files.data,
+	                       rec->files.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":exit_status");
+	rc = sqlite3_bind_text(stmt, idx, rec->exit_status.data,
+	                       rec->exit_status.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":diagnostics");
+	rc = sqlite3_bind_text(stmt, idx, rec->diagnostics.data,
+	                       rec->diagnostics.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":errors");
+	rc = sqlite3_bind_text(stmt, idx, rec->errors.data,
+	                       rec->errors.offset + 1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+	
+	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+	
+	idx = sqlite3_bind_parameter_index(stmt, ":machine");
+	if (rec->machine)
+		rc = sqlite3_bind_text(stmt, idx, rec->machine, -1, NULL);
+	else
+		rc = sqlite3_bind_null(stmt, idx);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	rc = sqlite3_step(stmt);
+	if (rc != SQLITE_DONE) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	sqlite3_finalize(stmt);
+	
+	/* Get the row id of the last inserted row */
+	mandb_rowid = sqlite3_last_insert_rowid(db);
+		
+/*------------------------Populate the mandb_meta table-----------------------*/
+	sqlstr = "INSERT INTO mandb_meta VALUES (:device, :inode, :mtime,"
+		 " :file, :md5_hash, :id)";
+	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+	if (rc != SQLITE_OK)
+		goto Out;
+
+	idx = sqlite3_bind_parameter_index(stmt, ":device");
+	rc = sqlite3_bind_int64(stmt, idx, rec->device);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":inode");
+	rc = sqlite3_bind_int64(stmt, idx, rec->inode);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+	rc = sqlite3_bind_int64(stmt, idx, rec->mtime);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":file");
+	rc = sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+	rc = sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":id");
+	rc = sqlite3_bind_int64(stmt, idx, mandb_rowid);
+	if (rc != SQLITE_OK) {
+		sqlite3_finalize(stmt);
+		goto Out;
+	}
+
+	rc = sqlite3_step(stmt);
+	sqlite3_finalize(stmt);
+	if (rc == SQLITE_CONSTRAINT) {
+		/* The *most* probable reason for reaching here is that
+		 * the UNIQUE contraint on the file column of the mandb_meta
+		 * table was violated.
+		 * This can happen when a file was updated/modified.
+		 * To fix this we need to do two things:
+		 * 1. Delete the row for the older version of this file
+		 *    from mandb table.
+		 * 2. Run an UPDATE query to update the row for this file
+		 *    in the mandb_meta table.
+		 */
+		warnx("Trying to update index for %s", rec->file_path);
+		char *sql = sqlite3_mprintf("DELETE FROM mandb "
+					    "WHERE rowid = (SELECT id"
+					    "  FROM mandb_meta"
+					    "  WHERE file = %Q)",
+					    rec->file_path);
+		sqlite3_exec(db, sql, NULL, NULL, &errmsg);
+		sqlite3_free(sql);
+		if (errmsg != NULL) {
+			warnx("%s", errmsg);
+			free(errmsg);
+		}
+		sqlstr = "UPDATE mandb_meta SET device = :device,"
+			 " inode = :inode, mtime = :mtime, id = :id,"
+			 " md5_hash = :md5 WHERE file = :file";
+		rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+		if (rc != SQLITE_OK) {
+			warnx("Update failed with error: %s",
+			    sqlite3_errmsg(db));
+			close_db(db);
+			cleanup(rec);
+			errx(EXIT_FAILURE,
+			    "Consider running makemandb with -f option");
+		}
+
+		idx = sqlite3_bind_parameter_index(stmt, ":device");
+		sqlite3_bind_int64(stmt, idx, rec->device);
+		idx = sqlite3_bind_parameter_index(stmt, ":inode");
+		sqlite3_bind_int64(stmt, idx, rec->inode);
+		idx = sqlite3_bind_parameter_index(stmt, ":mtime");
+		sqlite3_bind_int64(stmt, idx, rec->mtime);
+		idx = sqlite3_bind_parameter_index(stmt, ":id");
+		sqlite3_bind_int64(stmt, idx, mandb_rowid);
+		idx = sqlite3_bind_parameter_index(stmt, ":md5");
+		sqlite3_bind_text(stmt, idx, rec->md5_hash, -1, NULL);
+		idx = sqlite3_bind_parameter_index(stmt, ":file");
+		sqlite3_bind_text(stmt, idx, rec->file_path, -1, NULL);
+		rc = sqlite3_step(stmt);
+		sqlite3_finalize(stmt);
+
+		if (rc != SQLITE_DONE) {
+			warnx("%s", sqlite3_errmsg(db));
+			close_db(db);
+			cleanup(rec);
+			errx(EXIT_FAILURE,
+			    "Consider running makemandb with -f option");
+		}
+	} else if (rc != SQLITE_DONE) {
+		/* Otherwise make this error fatal */
+		warnx("Failed at %s\n%s", rec->file_path, sqlite3_errmsg(db));
+		cleanup(rec);
+		close_db(db);
+		exit(EXIT_FAILURE);
+	}
+
+/*------------------------ Populate the mandb_links table---------------------*/
+	char *str = NULL;
+	char *links;	
+	if (rec->links && strlen(rec->links)) {
+		links = rec->links;
+		for(ln = strtok(links, " "); ln; ln = strtok(NULL, " ")) {
+			if (ln[0] == ',')
+				ln++;
+			if(ln[strlen(ln) - 1] == ',')
+				ln[strlen(ln) - 1] = 0;
+			
+			str = sqlite3_mprintf("INSERT INTO mandb_links"
+					      " VALUES (%Q, %Q, %Q, %Q)",
+					      ln, rec->name, rec->section,
+					      rec->machine);
+			sqlite3_exec(db, str, NULL, NULL, &errmsg);
+			sqlite3_free(str);
+			if (errmsg != NULL) {
+				warnx("%s", errmsg);
+				cleanup(rec);
+				free(errmsg);
+				return -1;
+			}
+		}
+	}
+
+	cleanup(rec);
+	return 0;
+
+  Out:
+	warnx("%s", sqlite3_errmsg(db));
+	cleanup(rec);
+	return -1;
+}
+
+/*
+ * check_md5--
+ *  Generates the md5 hash of the file and checks if it already doesn't exist
+ *  in the table (passed as the 3rd parameter).
+ *  This function is being used to avoid hardlinks.
+ *  On successful completion it will also set the value of the fourth parameter
+ *  to the md5 hash of the file (computed previously). It is the responsibility
+ *  of the caller to free this buffer.
+ *  Return values:
+ *  -1: If an error occurs somewhere and sets the md5 return buffer to NULL.
+ *  0: If the md5 hash does not exist in the table.
+ *  1: If the hash exists in the database.
+ */
+static int
+check_md5(const char *file, sqlite3 *db, const char *table, char **buf)
+{
+	int rc = 0;
+	int idx = -1;
+	char *sqlstr = NULL;
+	sqlite3_stmt *stmt = NULL;
+
+	assert(file != NULL);
+	*buf = MD5File(file, NULL);
+	if (*buf == NULL) {
+		warn("md5 failed: %s", file);
+		return -1;
+	}
+
+	easprintf(&sqlstr, "SELECT * FROM %s WHERE md5_hash = :md5_hash",
+	    table);
+	rc = sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL);
+	if (rc != SQLITE_OK) {
+		free(sqlstr);
+		free(*buf);
+		*buf = NULL;
+		return -1;
+	}
+
+	idx = sqlite3_bind_parameter_index(stmt, ":md5_hash");
+	rc = sqlite3_bind_text(stmt, idx, *buf, -1, NULL);
+	if (rc != SQLITE_OK) {
+		warnx("%s", sqlite3_errmsg(db));
+		sqlite3_finalize(stmt);
+		free(sqlstr);
+		free(*buf);
+		*buf = NULL;
+		return -1;
+	}
+
+	if (sqlite3_step(stmt) == SQLITE_ROW) {
+		sqlite3_finalize(stmt);	
+		free(sqlstr);
+		return 0;
+	}
+
+	sqlite3_finalize(stmt);
+	free(sqlstr);
+	return 1;
+}
+
+/* Optimize the index for faster search */
+static void
+optimize(sqlite3 *db)
+{
+	const char *sqlstr;
+	char *errmsg = NULL;
+
+	if (mflags.verbosity)
+		printf("Optimizing the database index\n");
+	sqlstr = "INSERT INTO mandb(mandb) VALUES (\'optimize\');"
+		 "VACUUM";
+	sqlite3_exec(db, sqlstr, NULL, NULL, &errmsg);
+	if (errmsg != NULL) {
+		warnx("%s", errmsg);
+		free(errmsg);
+		return;
+	}
+}
+
+/* 
+ * cleanup --
+ *  cleans up the global buffers
+ */
+static void
+cleanup(mandb_rec *rec)
+{
+	rec->desc.offset = 0;
+	rec->lib.offset = 0;
+	rec->return_vals.offset = 0;
+	rec->env.offset = 0;
+	rec->exit_status.offset = 0;
+	rec->diagnostics.offset = 0;
+	rec->errors.offset = 0;
+	rec->files.offset = 0;
+
+	free(rec->machine);
+	rec->machine = NULL;
+
+	free(rec->links);
+	rec->links = NULL;
+
+	free(rec->file_path);
+	rec->file_path = NULL;
+
+	free(rec->name);
+	rec->name = NULL;
+
+	free(rec->name_desc);
+	rec->name_desc = NULL;
+
+	free(rec->md5_hash);
+	rec->md5_hash = NULL;
+}
+
+/*
+ * init_secbuffs--
+ *  Sets the value of buflen for all the sec_buff field of rec. And then
+ *  allocate memory to each sec_buff member of rec.
+ */
+static void
+init_secbuffs(mandb_rec *rec)
+{
+	/*
+	 * Some sec_buff might need more memory, for example desc,
+	 * which stores the data of the DESCRIPTION section,
+	 * while some might need very small amount of memory.
+	 * Therefore explicitly setting the value of buflen field for
+	 * each sec_buff.
+	 */
+	rec->desc.buflen = 10 * BUFLEN;
+	rec->desc.data = emalloc(rec->desc.buflen);
+	rec->desc.offset = 0;
+
+	rec->lib.buflen = BUFLEN / 2;
+	rec->lib.data = emalloc(rec->lib.buflen);
+	rec->lib.offset = 0;
+
+	rec->return_vals.buflen = BUFLEN;
+	rec->return_vals.data = emalloc(rec->return_vals.buflen);
+	rec->return_vals.offset = 0;
+
+	rec->exit_status.buflen = BUFLEN;
+	rec->exit_status.data = emalloc(rec->exit_status.buflen);
+	rec->exit_status.offset = 0;
+
+	rec->env.buflen = BUFLEN;
+	rec->env.data = emalloc(rec->env.buflen);
+	rec->env.offset = 0;
+
+	rec->files.buflen = BUFLEN;
+	rec->files.data = emalloc(rec->files.buflen);
+	rec->files.offset = 0;
+
+	rec->diagnostics.buflen = BUFLEN;
+	rec->diagnostics.data = emalloc(rec->diagnostics.buflen);
+	rec->diagnostics.offset = 0;
+
+	rec->errors.buflen = BUFLEN;
+	rec->errors.data = emalloc(rec->errors.buflen);
+	rec->errors.offset = 0;
+}
+
+/*
+ * free_secbuffs--
+ *  This function should be called at the end, when all the pages have been
+ *  parsed.
+ *  It frees the memory allocated to sec_buffs by init_secbuffs in the starting.
+ */
+static void
+free_secbuffs(mandb_rec *rec)
+{
+	free(rec->desc.data);
+	free(rec->lib.data);
+	free(rec->return_vals.data);
+	free(rec->exit_status.data);
+	free(rec->env.data);
+	free(rec->files.data);
+	free(rec->diagnostics.data);
+	free(rec->errors.data);
+}
+
+static char *
+parse_escape(const char *str)
+{
+	const char *backslash, *last_backslash;
+	char *result, *iter;
+	size_t len;
+
+	assert(str);
+
+	last_backslash = str;
+	backslash = strchr(str, '\\');
+	if (backslash == NULL)
+		return estrdup(str);
+
+	result = emalloc(strlen(str) + 1);
+	iter = result;
+
+	do {
+		len = backslash - last_backslash;
+		memcpy(iter, last_backslash, len);
+		iter += len;
+		if (backslash[1] == '-' || backslash[1] == ' ') {
+			*iter++ = backslash[1];
+			last_backslash = backslash + 2;
+			backslash = strchr(backslash + 2, '\\');
+		} else {
+			++backslash;
+			mandoc_escape(&backslash, NULL, NULL);
+			last_backslash = backslash;
+			if (backslash == NULL)
+				break;
+			backslash = strchr(last_backslash, '\\');
+		}
+	} while (backslash != NULL);
+	if (last_backslash != NULL)
+		strcpy(iter, last_backslash);
+	iter = result;
+	while ((iter = strchr(iter, ASCII_HYPH)) != NULL)
+		*iter = '-';
+	return result;
+}
+
+/*
+ * append--
+ *  Concatenates a space and src at the end of sbuff->data (much like concat in
+ *  apropos-utils.c).
+ *  Rather than reallocating space for writing data, it uses the value of the
+ *  offset field of sec_buff to write new data at the free space left in the
+ *  buffer.
+ *  In case the size of the data to be appended exceeds the number of bytes left
+ *  in the buffer, it reallocates buflen number of bytes and then continues.
+ *  Value of offset field should be adjusted as new data is written.
+ *
+ *  NOTE: This function does not write the null byte at the end of the buffers,
+ *  write a null byte at the position pointed to by offset before inserting data
+ *  in the db.
+ */
+static void
+append(secbuff *sbuff, const char *src)
+{
+	short flag = 0;
+	size_t srclen, newlen;
+	char *temp;
+
+	assert(src != NULL);
+	temp = parse_escape(src);
+	srclen = strlen(temp);
+
+	if (sbuff->data == NULL) {
+		sbuff->data = emalloc(sbuff->buflen);
+		sbuff->offset = 0;
+	}
+
+	newlen = sbuff->offset + srclen + 2;
+	if (newlen >= sbuff->buflen) {
+		while (sbuff->buflen < newlen)
+			sbuff->buflen += sbuff->buflen;
+		sbuff->data = erealloc(sbuff->data, sbuff->buflen);
+		flag = 1;
+	}
+
+	/* Append a space at the end of the buffer. */
+	if (sbuff->offset || flag)
+		sbuff->data[sbuff->offset++] = ' ';
+	/* Now, copy src at the end of the buffer. */
+	memcpy(sbuff->data + sbuff->offset, temp, srclen);
+	sbuff->offset += srclen;
+	free(temp);
+}
+
+static void
+usage(void)
+{
+	fprintf(stderr, "Usage: %s [-flo]\n", getprogname());
+	exit(1);
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/run_query.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,223 @@
+.\" $NetBSD: run_query.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY 3
+.Os
+.Sh NAME
+.Nm run_query
+.Nd run a query against
+.Pa /var/db/man.db
+and process the results in a callback function.
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query "sqlite3 *db" "const char **snippet_args" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query
+function prepares the user supplied query in a form suitable to be run
+against
+.Pa /var/db/man.db
+and executes the query.
+For each row obtained in the result set,
+.Fn run_query
+will call the user supplied callback function, which should contain the
+logic for processing the data thus obtained.
+.Pp
+The
+.Fn run_query
+function takes following arguments:
+.Bl -tag -width 8n
+.It Fa sqlite3 *db
+Handle to the database connection which can be obtained by calling
+.Fn init_db .
+.It Fa const char *snippet_args
+An array of strings which specify the
+delimiters to the matching text in snippet.
+It is an optional argument and caller can supply a
+.Dv NULL
+value for it, in which case, a default value of
+.Brq \&"\&", \&"\&", \&"...\&"
+will be used.
+The 3 members of the array specify the following values:
+.Bl -enum -offset indent
+.It
+The first element marks the beginning of each matching token in the snippet.
+.It
+The second element the end of each matching token in the snippet.
+.It
+The third element is used to delimit the beginning and end of the snippet.
+.El
+For example for highlighting matching tokens in HTML style mark-up, use this
+value:
+.Bd -literal -offset indent
+ const char **snippet_args = {
+	"<b>",
+	"</b>",
+	"..."
+ };
+.Ed
+.It Fa query_args *args
+.Ft query_args
+is a struct which is defined in
+.Pa apropos-utils.h .
+It has the following fields:
+.Bl -tag -width 8n -offset indent
+.It Li const char *search_str
+This is the query as entered by the user.
+You may want to pre-process it to do sanitization etc.
+.It Li int *sec_nums
+This is an array of
+.Ft int
+wherein each index element represents the
+section number in which search should be performed.
+The sections whose corresponding index element in this array is set to
+.Dv NULL
+are not searched.
+If all the elements in the array are
+.Dv 0
+then all the sections are searched.
+Alternatively pass
+.Dv NULL
+as to search in all the sections.
+.It Li int nrec
+This specifies the number of matching rows to fetch from the database.
+Specifiy a negative value to fetch all the matching rows.
+.It Li int offset
+This specifies the offset within the result-set. Use it to specify the position
+from where to start processing the result-set.
+For example if the value of nrec is m and value of offset is n, then the first
+n records from the result-set will be omitted and rest m-n (or less) records will
+be available for processing inside the callback.
+Use a negative value if you don't wish to offset any records.
+.It Li const char *machine
+The machine architecture to which the searches should be restricted.
+Specify NULL if the search should not be restricted a particular machine architecture.
+.It Li int (*callback) (void *, const char *, const char *, const char *,\
+const char *, size_t)
+This is the callback function which will
+be called for each row retrieved from the database.
+The function should return a value of 0 on successful execution.
+A non-zero return value will indicate a failure and
+.Fn run_query
+will return.
+The interface of the callback function is described later in this section.
+.It Li void *callback_data
+Use this argument to pass any data to the callback function.
+It can be retrieved inside the callback function from its first argument.
+.It Li char **errmsg
+If an error occurs while fetching the data from the database,
+a human readable error message will be stored in
+.Fa errmsg .
+If no error occurs then
+.Fa errmsg
+will be set to
+.Dv NULL .
+In case
+.Fa errmsg
+is not
+.Dv NULL ,
+then the caller should make sure to free it.
+.El
+.El
+.Pp
+The interface of the callback function is
+.Ft int
+.Fn (*callback) "void *callback_data" "const char *section" "const char *name"\
+"const char *name_desc" "const char *snippet" "size_t snippet_length"
+.Pp
+Its arguments are:
+.Bl -column -offset indent "Function" "Argument Description"
+.It Li void *callback_data Ta This is the caller supplied data.
+.It Li const char *section Ta Ta \&It is the section number to which this man
+page belongs.
+.It Li const char *name Ta This is the name of the man page
+.It Li const char *name_desc Ta This is the one line description of this man
+page obtained from it's NAME section.
+.It Li const char *snippet Ta This is a snippet of the matching text from this
+man page.
+.It Li size_t snippet_length Ta This is the length of the snippet.
+.El
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width /var/db/man.db -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Pa apropos.c
+uses
+.Fn run_query .
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, NULL, &args) < 0)
+		errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+	const char *name_desc, const char *snippet, size_t snippet_length )
+{
+	/* The user supplied data could be obtained as follows */
+	// my_data *buf = (my_data *) data;
+
+	fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+		snippet);
+	return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query_html 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/run_query_html.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,116 @@
+.\" $NetBSD: run_query_html.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_HTML 3
+.Os
+.Sh NAME
+.Nm run_query_html
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in HTML format
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_html "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_html
+is very much similar to
+.Fn run_query
+but with the following three differences.
+.Bl -enum -offset indent
+.It
+.Nm
+takes only two arguments.
+.It
+The snippet returned by
+.Nm
+uses <b> and </b> tags to highlight the matching pieces of text so that it
+appears bold in a web browser.
+.It
+.Nm
+will do limited quoting of the inline HTML fragments in the snippet.
+Concretely, it will replace <, >, ", & with &lt; &gt; &quot; and &amp;
+respectively.
+.El
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_html
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = 10;
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = NULL;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+		errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+	const char *name_desc, const char *snippet, size_t snippet_length)
+{
+	fprintf(stdout, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+		snippet);
+	return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_pager 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/run_query_pager.3	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,106 @@
+.\" $NetBSD: run_query_pager.3,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2011 Abhinav Upadhyay <er.abhinav.upadhyay@gmail.com>
+.\" All rights reserved.
+.\"
+.\" This code was developed as part of Google's Summer of Code 2011 program.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd December 03, 2011
+.Dt RUN_QUERY_PAGER 3
+.Os
+.Sh NAME
+.Nm run_query_pager
+.Nd run a query against
+.Pa /var/db/man.db
+and returns the results in a format which can be piped to a pager
+.Sh SYNOPSIS
+.In apropos-utils.h
+.Ft int
+.Fn run_query_pager "sqlite3 *db" "query_args *args"
+.Sh DESCRIPTION
+The
+.Fn run_query_pager
+is very similar to
+.Xr run_query_html 3
+with the only difference that the snippet returned is formatted in a manner so
+that the matching text in it appears highlighted when viewed using a pager.
+.Pp
+For more details, refer to the manual page of
+.Xr run_query 3 .
+.Sh RETURN VALUES
+On successful execution the
+.Fn run_query_pager
+function will return 0 and in case of an error \-1 will be returned.
+.Sh FILES
+.Bl -hang -width -compact
+.It Pa /var/db/man.db
+The Sqlite FTS database which contains an index of the manual pages.
+.El
+.Sh EXAMPLES
+Following is a code excerpt of how
+.Nm
+can be used.
+.Bd -literal -offset indent
+#include <apropos-utils.h>
+query_args args;
+FILE *pager = popen("more", "w");
+char *errmsg = NULL;
+int *sec_nums = {0, 1, 1, 0, 0, 0, 0, 0, 0};
+args.search_str = argv[1];
+args.sec_nums = sec_nums;
+args.nrec = -1;	//A negative value means fetch all matching records
+args.offset = -1;
+args.machine = NULL;
+args.callback = &query_callback;
+args.callback_data = (void *)pager;
+args.errmsg = &errmsg;
+if (run_query(db, &args) < 0)
+		errx(EXIT_FAILURE, "%s", errmsg);
+}
+
+free(query);
+free(errmsg);
+pclose(pager);
+
+static int
+query_callback(void *data, const char *section, const char *name,
+	const char *name_desc, const char *snippet, size_t snippet_length)
+{
+	FILE *out = (FILE *) data;
+	fprintf(out, "%s(%s)\t%s\en%s\en\en", name, section, name_desc,
+		snippet);
+	return 0;
+}
+.Ed
+.Sh SEE ALSO
+.Xr apropos-utils 3 ,
+.Xr close_db 3 ,
+.Xr init_db 3 ,
+.Xr run_query 3 ,
+.Xr run_query_html 3
+.Sh AUTHORS
+.An Abhinav Upadhyay
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/stopwords.txt	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,251 @@
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+about
+again
+all
+also
+always
+an
+and
+another
+any
+are
+around
+as
+ask
+at
+b
+back
+be
+because
+been
+before
+below
+between
+but
+by
+bye
+can
+case
+consist
+could
+d
+did
+does
+down
+e
+each
+early
+either
+end
+enough
+even
+every
+f
+fact
+far
+few
+follow
+four
+from
+full
+further
+g
+general
+get
+give
+given
+good
+got
+great
+h
+had
+has
+have
+having
+here
+high
+him
+his
+how
+however
+i
+if
+important
+in
+interest
+into
+is
+it
+j
+just
+k
+keep
+keeps
+kind
+knew
+know
+l
+large
+larger
+last
+later
+latest
+latter
+least
+let
+like
+likely
+long
+longer
+m
+made
+many
+may
+me
+might
+most
+mostly
+much
+must
+my
+n
+names
+necessary
+need
+needs
+never
+new
+next
+no
+non
+noone
+not
+nothing
+o
+of
+off
+often
+old
+older
+on
+once
+only
+or
+order
+our
+out
+over
+p
+part
+per
+perhaps
+possible
+present
+problem
+q
+quite
+r
+rather
+really
+right
+room
+s
+said
+same
+saw
+say
+says
+second
+see
+seem
+seemed
+seems
+sees
+several
+shall
+should
+side
+sides
+small
+smaller
+so
+some
+something
+state
+states
+still
+such
+sure
+t
+take
+taken
+that
+the
+their
+them
+then
+there
+therefore
+these
+thing
+think
+thinks
+this
+those
+though
+three
+thus
+to
+together
+too
+took
+toward
+turn
+two
+u
+until
+up
+upon
+us
+use
+used
+uses
+v
+very
+w
+want
+wanted
+wants
+was
+way
+ways
+we
+well
+went
+were
+what
+when
+whether
+why
+will
+willing
+with
+within
+without
+work
+would
+x
+y
+year
+yet
+you
+z
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/whatis.1	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,58 @@
+.\" $NetBSD: whatis.1,v 1.1 2012/02/07 19:13:32 joerg Exp $
+.\"
+.\" Copyright (c) 2012 Joerg Sonnenberger <joerg@NetBSD.org>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\"
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in
+.\"    the documentation and/or other materials provided with the
+.\"    distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+.\" ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+.\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+.\" FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+.\" COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+.\" INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+.\" BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+.\" LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+.\" AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.Dd February 1, 2012
+.Dt WHATIS 1
+.Os
+.Sh NAME
+.Nm whatis
+.Nd describe what a command is
+.Sh SYNOPSIS
+.Nm
+.Ar command Ar ...
+.Sh DESCRIPTION
+The
+.Nm
+program queries the database
+.Pa /var/db/man.db
+for manual pages with name
+.Ar command
+and prints the name of the manual page, the section and the description from
+the section NAME.
+.Sh SEE ALSO
+.Xr apropos 1 ,
+.Xr makemandb 8
+.Sh HISTORY
+The
+.Nm
+command appeared in 3.0BSD.
+It was rewritten in
+.Nx 6.0
+to use the SQLite index created by
+.Xr makemandb 8 .
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/usr.sbin/makemandb/whatis.c	Tue Feb 07 19:13:24 2012 +0000
@@ -0,0 +1,101 @@
+/*	$NetBSD	*/
+/*-
+ * Copyright (c) 2012 Joerg Sonnenberger <joerg@NetBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
+ * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__RCSID("$NetBSD: whatis.c,v 1.1 2012/02/07 19:13:32 joerg Exp $");
+
+#include <err.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "apropos-utils.h"
+
+__dead static void
+usage(void)
+{
+	fprintf(stderr, "%s ...\n", "whatis");
+	exit(EXIT_FAILURE);
+}
+
+static int
+whatis(sqlite3 *db, const char *cmd)
+{
+	static const char sqlstr[] = "SELECT name, section, name_desc"
+				     " FROM mandb WHERE name=?"
+				     " ORDER BY section, name";
+	sqlite3_stmt *stmt = NULL;
+	int retval;
+
+	if (sqlite3_prepare_v2(db, sqlstr, -1, &stmt, NULL) != SQLITE_OK)
+		errx(EXIT_FAILURE, "Unable to query database");
+	if (sqlite3_bind_text(stmt, 1, cmd, -1, NULL) != SQLITE_OK)
+		errx(EXIT_FAILURE, "Unable to query database");
+	retval = 1;
+	while (sqlite3_step(stmt) == SQLITE_ROW) {
+		printf("%s(%s) - %s\n", sqlite3_column_text(stmt, 0),
+		    sqlite3_column_text(stmt, 1),
+		    sqlite3_column_text(stmt, 2));
+		retval = 0;
+	}
+	sqlite3_finalize(stmt);
+	if (retval)
+		fprintf(stderr, "%s: not found\n", cmd);
+	return retval;
+}
+
+int
+main(int argc, char *argv[])
+{
+	sqlite3 *db;
+	int ch, retval;
+
+	while ((ch = getopt(argc, argv, "")) != -1) {
+		switch (ch) {
+		default:
+			usage();
+		}
+	}
+	argc -= optind;
+	argv += optind;
+
+	if (argc == 0)
+		usage();
+
+	if ((db = init_db(MANDB_READONLY)) == NULL)
+		exit(EXIT_FAILURE);
+
+	retval = 0;
+	while (argc--)
+		retval |= whatis(db, *argv++);
+
+	close_db(db);
+	return retval;
+}