5 files changed, 267 insertions, 53 deletions

diff --git a/Mk/Scripts/check-stagedir.sh b/Mk/Scripts/check-stagedir.sh
index e2085718be27..0f7af97956c4 100644
--- a/Mk/Scripts/check-stagedir.sh
+++ b/Mk/Scripts/check-stagedir.sh
@@ -1,9 +1,19 @@
 #!/bin/sh
 # ports/Mk/Scripts/check-stagedir.sh - called from ports/Mk/bsd.stage.mk
 # $FreeBSD$
+#
+# MAINTAINER: portmgr@FreeBSD.org
+#
+# This script serves 2 purposes:
+# 1. Generate a plist
+# 2. Test a plist for issues:
+#  a. Files in STAGEDIR that are missing from plist
+#  b. Files in plist missing from STAGEDIR
+#  c. Files in plist which are owned by dependencies/MTREEs
 
 set -e
 export LC_ALL=C
+ret=0
 
 # lists an mtree file's contents, prefixed to dir.
 listmtree() { # mtreefile prefix
@@ -16,15 +26,16 @@ listmtree() { # mtreefile prefix
 # obtain operating mode from command line
 makeplist=0
 case "$1" in
-	orphans)	;;
+	checkplist)	;;
 	makeplist)	makeplist=1 ;;
-	*) echo >&2 "Usage: $0 {orphans|makelist}" ; exit 1 ;;
+	*) echo >&2 "Usage: $0 {checkplist|makelist}" ; exit 1 ;;
 esac
 
 # validate environment
 envfault=
 for i in STAGEDIR PREFIX LOCALBASE WRKDIR WRKSRC MTREE_FILE GNOME_MTREE_FILE \
-    TMPPLIST DOCSDIR EXAMPLESDIR PLIST_SUB
+    TMPPLIST PLIST_SUB_SED SCRIPTSDIR PACKAGE_DEPENDS WITH_PKGNG PKG_QUERY \
+    PORT_OPTIONS NO_PREFIX_RMDIR
 do
     if ! ( eval ": \${${i}?}" ) 2>/dev/null ; then
 		envfault="${envfault}${envfault:+" "}${i}"
@@ -43,8 +54,23 @@ set -u
 #    files to the pipe and dirs to a separate file.
 if [ $makeplist = 0 ] ; then
 	# check for orphans
+	echo "===> Checking for items in STAGEDIR missing from pkg-plist"
+
 	cwd=${PREFIX}
 	while read line; do
+		# Handle deactivated OPTIONS. Treat "@comment file" as being in
+		# the plist so it does not show up as an orphan. PLIST_SUB uses
+		# a @comment to deactive files. XXX: It would be better to
+		# make all ports use @ignore instead of @comment.
+		comment=
+		if [ ${makeplist} -eq 0 -a -z "${line%%@comment *}" ]; then
+			line="${line#@comment }"
+			# Remove @comment so it can be parsed as a file,
+			# but later prepend it again to create a list of
+			# all files commented and uncommented.
+			comment="@comment "
+		fi
+
 		case $line in
 		@dirrm*|'@unexec rmdir'*)
 			line="$(printf %s "$line" \
@@ -55,25 +81,49 @@ if [ $makeplist = 0 ] ; then
 				-e 's/@dirrm(try)?[[:space:]]+//' \
 				-e 's/[[:space:]]+$//')"
 			case "$line" in
-			/*) echo >&3 "$line" ;;
-			*)  echo >&3 "$cwd/$line" ;;
+			/*) echo >&3 "${comment}${line%/}" ;;
+			*)  echo >&3 "${comment}${cwd}/${line%/}" ;;
 			esac
 		;;
-		@info*)
+		# Handle [file] Keywords
+		@info\ *|@sample\ *)
+			set -- $line
+			shift
+			echo "${comment}${cwd}/$@"
+		;;
+		# Handle [dirrmty] Keywords
+		@fc\ *|@fcfontsdir\ *|@fontsdir\ *)
 			set -- $line
 			shift
-			echo "$cwd/$@"
+			echo "${comment}@dirrmtry ${cwd}/$@"
 		;;
+
 		# order matters here - we must check @cwd first because
 		# otherwise the @cwd* would also match it first, shadowing the
 		# @cwd) line.
 		@cwd|@cd) cwd=${PREFIX} ;;
-		@cwd*|@cd*) set -- $line ; cwd=$2 ;;
+		@cwd*|@cd*)
+			set -- $line
+			cwd=$2
+			# Don't set cwd=/ as it causes // in plist and
+			# won't match later.
+			[ "${cwd}" = "/" ] && cwd=
+			;;
 		@*) ;;
-		/*) echo "$line" ;;
-		*)  echo "$cwd/$line" ;;
+		/*) echo "${comment}${line}" ;;
+		*)  echo "${comment}${cwd}/${line}" ;;
 		esac
-	done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | sort >${WRKDIR}/.plist-files
+	done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | \
+	    sort >${WRKDIR}/.plist-files
+	unset TMPPLIST
+	# Create the -no-comments files and trim out @comment from the plists.
+	# This is used for various tests later.
+	sed -e '/^@comment/d' ${WRKDIR}/.plist-dirs-unsorted \
+	    >${WRKDIR}/.plist-dirs-unsorted-no-comments
+	sed -i '' -e 's/^@comment //' ${WRKDIR}/.plist-dirs-unsorted
+	sed -e '/^@comment/d' ${WRKDIR}/.plist-files \
+	    >${WRKDIR}/.plist-files-no-comments
+	sed -i '' -e 's/^@comment //' ${WRKDIR}/.plist-files
 else
 	# generate plist - pretend the plist had been empty
 	: >${WRKDIR}/.plist-dirs-unsorted
@@ -84,55 +134,173 @@ fi
 ### PRODUCE MTREE FILE
 {
 	listmtree /etc/mtree/BSD.root.dist ""
-	#listmtree /etc/mtree/BSD.usr.dist /usr
+	listmtree /etc/mtree/BSD.usr.dist /usr
 	listmtree /etc/mtree/BSD.var.dist /var
 
+	# Use MTREE_FILE if specified and it doesn't already match LOCALBASE
 	if [ -n "${MTREE_FILE}" ]; then
-		listmtree "${MTREE_FILE}" "${PREFIX}"
+		if [ "${PREFIX}" != "${LOCALBASE}" -o \
+		    "${MTREE_FILE}" != "${PORTSDIR}/Templates/BSD.local.dist" \
+		    ]; then
+			listmtree "${MTREE_FILE}" "${PREFIX}"
+		fi
 	fi
+	listmtree "${PORTSDIR}/Templates/BSD.local.dist" "${LOCALBASE}"
 
 	if [ -n "${GNOME_MTREE_FILE}" ]; then
 		listmtree "${GNOME_MTREE_FILE}" "${PREFIX}"
 	fi
+	unset MTREE_FILE GNOME_MTREE_FILE
 
-	a=${PREFIX}
-	while :; do
-		echo ${a}
-		a=${a%/*}
-		[ -z "${a}" ] && break
-	done
-} > ${WRKDIR}/.mtree
+	# Add in PREFIX if this port wants it
+	if [ ${NO_PREFIX_RMDIR} -eq 0 ]; then
+		a=${PREFIX}
+		while :; do
+			echo ${a}
+			a=${a%/*}
+			[ -z "${a}" ] && break
+		done
+	fi
+} >${WRKDIR}/.mtree
 
-for i in $PLIST_SUB
-do
-	echo $i
-done | awk -F= '{sub(/^"/, "", $2); sub(/"$/, "", $2); print length($2), $1, $2 | "sort -nr" }' | while read l k v
-do
-	if [ $l -gt 1 ]
-	then
-		echo "s,${v},%%${k}%%,g;"
+### GATHER DIRS OWNED BY RUN-DEPENDS. WHY ARE WE SCREAMING?
+: >${WRKDIR}/.run-depends-dirs
+if [ -n "${WITH_PKGNG}" ]; then
+	echo "${PACKAGE_DEPENDS}" | xargs ${PKG_QUERY} "%D" | \
+	    sed -e 's,/$,,' | sort -u >>${WRKDIR}/.run-depends-dirs
+else
+	# Evaluate ACTUAL-PACKAGE-DEPENDS
+	packagelist=
+	package_depends=$(eval ${PACKAGE_DEPENDS})
+	if [ -n "${package_depends}" ]; then
+		# This ugly mess can go away with pkg_install EOL
+		awk_script=$(cat <<'EOF'
+			/Deinstall directory remove:/ {print $4}
+			/UNEXEC 'rmdir "[^"]*" 2>\/dev\/null \|\| true'/ {
+				gsub(/"%D\//, "\"", $0)
+				match($0, /"[^"]*"/)
+				dir=substr($0, RSTART+1, RLENGTH-2)
+				print dir
+			}
+EOF
+)
+		echo "${package_depends}" | while read line; do
+			${PKG_QUERY} -f ${line%%:*} | \
+			    awk "${awk_script}" | \
+			    sed -e "/^[^/]/s,^,${LOCALBASE}/,"
+		done | sort -u >>${WRKDIR}/.run-depends-dirs
 	fi
-done > ${WRKDIR}/.plist_sub
+fi
+unset PACKAGE_DEPENDS PKG_QUERY
+
+### HANDLE PORTDOCS/PORTEXAMPLES
+sed_portdocsexamples="/%%DOCSDIR%%/s!^!%%PORTDOCS%%!g; /%%EXAMPLESDIR%%/s!^!%%PORTEXAMPLES%%!g;"
+if [ ${makeplist} -eq 0 ]; then
+#	echo "=====> Using OPTIONS: ${PORT_OPTIONS}" | /usr/bin/fmt -w 79 | \
+#	    sed -e '2,$s/^/                      /'
+	# Handle magical PORT* features
+	for option in DOCS EXAMPLES; do
+		want_option=0
+		case " ${PORT_OPTIONS} " in
+		*\ ${option}\ *) want_option=1 ;;
+		esac
+		[ ${want_option} -eq 0 ] && \
+		    sed_portdocsexamples="${sed_portdocsexamples} /^%%PORT${option}%%/d;"
+	done
+	unset PORT_OPTIONS
+fi
 
-sed_plist_sub=`cat ${WRKDIR}/.plist_sub`
+sed_plist_sub=$(echo "${PLIST_SUB_SED}" | /bin/sh ${SCRIPTSDIR}/plist_sub_sed_sort.sh)
+unset PLIST_SUB_SED
+sed_files="s!${PREFIX}/!!g; ${sed_plist_sub} ${sed_portdocsexamples} \
+    /^share\/licenses/d;"
+
+sed_dirs="s!${PREFIX}/!!g; ${sed_plist_sub} s,^,@dirrmtry ,; \
+    ${sed_portdocsexamples} \
+    s!@dirrmtry \(/.*\)!@unexec rmdir \"\1\" >/dev/null 2>\&1 || :!; \
+    /^@dirrmtry share\/licenses/d;"
+
+# If checking orphans, send all output to a temp file so whitelisting can be
+# done
+: >${WRKDIR}/.staged-plist
 
 ### HANDLE FILES
-find ${STAGEDIR} -type f -o -type l | sort | sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files
-comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files \
-	| sed \
-	-e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \
-	-e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \
-	-e "s,${PREFIX}/,,g" \
-	-e "${sed_plist_sub}" | grep -v "^share/licenses" || [ $? = 1 ]
+find ${STAGEDIR} -type f -o -type l | sort | \
+    sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files
+comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files | \
+    sed -e "${sed_files}" \
+     >>${WRKDIR}/.staged-plist || :
 
 ### HANDLE DIRS
-cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree | sort -u >${WRKDIR}/.traced-dirs
-find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | sort >${WRKDIR}/.staged-dirs
+cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree \
+    ${WRKDIR}/.run-depends-dirs | sort -u >${WRKDIR}/.traced-dirs
+find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | \
+    sort >${WRKDIR}/.staged-dirs
 comm -13 ${WRKDIR}/.traced-dirs ${WRKDIR}/.staged-dirs \
-	| sort -r | sed \
-	-e 's,^,@dirrmtry ,' \
-	-e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \
-	-e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \
-	-e "s,${PREFIX}/,,g" \
-	-e "${sed_plist_sub}" \
-	-e 's,@dirrmtry \(/.*\),@unexec rmdir >/dev/null 2>\&1 \1 || :,' | grep -v "^@dirrmtry share/licenses" || [ $? = 1 ]
+    | sort -r | sed "${sed_dirs}" \
+    >>${WRKDIR}/.staged-plist || :
+
+# If just making plist, show results and exit successfully.
+if [ ${makeplist} -eq 1 ]; then
+	cat ${WRKDIR}/.staged-plist
+	exit 0
+fi
+
+# Handle whitelisting
+while read path; do
+	case "${path}" in
+	*.bak) ;;
+	*.orig) ;;
+	#*/info/dir|info/dir) ;;
+	*)
+		# An orphan was found, return non-zero status
+		ret=1
+		echo "Error: Orphaned: ${path}" >&2
+	;;
+	esac
+done < ${WRKDIR}/.staged-plist
+
+sort -u ${WRKDIR}/.plist-dirs-unsorted-no-comments \
+    >${WRKDIR}/.plist-dirs-sorted-no-comments
+
+# Anything listed in plist and in restricted-dirs is a failure. I.e.,
+# it's owned by a run-time dependency or one of the MTREEs.
+echo "===> Checking for directories owned by dependencies or MTREEs"
+cat ${WRKDIR}/.mtree ${WRKDIR}/.run-depends-dirs | sort -u \
+    >${WRKDIR}/.restricted-dirs
+: >${WRKDIR}/.invalid-plist-dependencies
+comm -12 ${WRKDIR}/.plist-dirs-sorted-no-comments ${WRKDIR}/.restricted-dirs \
+    | sort -r | sed "${sed_dirs}" \
+    >>${WRKDIR}/.invalid-plist-dependencies || :
+if [ -s "${WRKDIR}/.invalid-plist-dependencies" ]; then
+	ret=1
+	while read line; do
+		echo "Error: Owned by dependency: ${line}" >&2
+	done < ${WRKDIR}/.invalid-plist-dependencies
+fi
+
+echo "===> Checking for items in pkg-plist which are not in STAGEDIR"
+: >${WRKDIR}/.invalid-plist-missing
+comm -23 ${WRKDIR}/.plist-files-no-comments ${WRKDIR}/.staged-files | \
+    sed -e "${sed_files}" \
+    >>${WRKDIR}/.invalid-plist-missing || :
+
+comm -23 ${WRKDIR}/.plist-dirs-sorted-no-comments ${WRKDIR}/.staged-dirs \
+    | sort -r | sed "${sed_dirs}" \
+    >>${WRKDIR}/.invalid-plist-missing || :
+if [ -s "${WRKDIR}/.invalid-plist-missing" ]; then
+	ret=1
+	while read line; do
+		echo "Error: Missing: ${line}" >&2
+	done < ${WRKDIR}/.invalid-plist-missing
+fi
+
+if [ ${ret} -ne 0 ]; then
+	echo "===> Error: Plist issues found." >&2
+	if [ "${PREFIX}" != "${LOCALBASE}" ]; then
+	    echo "===> Warning: Test was done with PREFIX != LOCALBASE"
+	    echo "===> Warning: The port may not be properly installing into PREFIX"
+	fi
+fi
+
+exit ${ret}
diff --git a/Mk/Scripts/check_leftovers.sh b/Mk/Scripts/check_leftovers.sh
index a85ac5d13dd4..48e704464c02 100644
--- a/Mk/Scripts/check_leftovers.sh
+++ b/Mk/Scripts/check_leftovers.sh
@@ -41,7 +41,7 @@ else
 	LOCALBASE=$(make -C ${portdir} -VLOCALBASE)
 fi
 homedirs=$(awk -F: -v users=$(make -C ${portdir} -V USERS|sed -e 's, ,|,g;/^$/d') 'users && $1 ~ users {print $9}' ${PORTSDIR}/UIDs|sort -u|sed -e "s|/usr/local|${PREFIX}|")
-plistsub_sed=$(make -C ${portdir} -VPLIST_SUB_SED)
+plistsub_sed=$(make -C ${portdir} -VPLIST_SUB_SED | /bin/sh ${PORTSDIR}/Mk/Scripts/plist_sub_sed_sort.sh)
 tmpplist=$(make -C ${portdir} -VTMPPLIST)
 
 while read modtype path extra; do
diff --git a/Mk/Scripts/plist_sub_sed_sort.sh b/Mk/Scripts/plist_sub_sed_sort.sh
new file mode 100644
index 000000000000..aa58c73e883c
--- /dev/null
+++ b/Mk/Scripts/plist_sub_sed_sort.sh
@@ -0,0 +1,17 @@
+#! /bin/sh
+# $FreeBSD$
+#
+# MAINTAINER: portmgr@FreeBSD.org
+#
+# PLIST_SUB_SED helper to sort by longest value first.
+
+exec awk '{
+	while (match($0, /s![^!]*![^!]*!g;/)) {
+		sedp=substr($0, RSTART, RLENGTH)
+		$0=substr($0, RSTART+RLENGTH)
+		split(sedp, a, "!")
+		# Convert \. to . for sorting.
+		gsub(/\\./, ".", a[2])
+		print length(a[2]), sedp
+	}
+}' | sort -rn | awk '{print $2}' | paste -s -d ' ' -
diff --git a/Mk/bsd.options.mk b/Mk/bsd.options.mk
index f326f4938c99..5130a813babc 100644
--- a/Mk/bsd.options.mk
+++ b/Mk/bsd.options.mk
@@ -139,14 +139,24 @@ _OPTIONS_DEPENDS=	PKG FETCH EXTRACT PATCH BUILD LIB RUN
 # Set the default values for the global options, as defined by portmgr
 .if !defined(NOPORTDOCS)
 PORT_OPTIONS+=	DOCS
+.else
+OPTIONS_WARNINGS+= "NOPORTDOCS"
+WITHOUT+=			DOCS
+OPTIONS_WARNINGS_UNSET+=	DOCS
 .endif
 
 .if !defined(WITHOUT_NLS)
 PORT_OPTIONS+=	NLS
+.else
+WITHOUT+=		NLS
 .endif
 
 .if !defined(NOPORTEXAMPLES)
 PORT_OPTIONS+=	EXAMPLES
+.else
+OPTIONS_WARNINGS+= "NOPORTEXAMPLES"
+WITHOUT+=			EXAMPLES
+OPTIONS_WARNINGS_UNSET+=	EXAMPLES
 .endif
 
 PORT_OPTIONS+=	IPV6
diff --git a/Mk/bsd.stage.mk b/Mk/bsd.stage.mk
index e643635556d0..da0f53146458 100644
--- a/Mk/bsd.stage.mk
+++ b/Mk/bsd.stage.mk
@@ -24,9 +24,23 @@ CO_ENV+=	STAGEDIR=${STAGEDIR} \
 		MTREE_FILE=${MTREE_FILE} \
 		GNOME_MTREE_FILE=${GNOME_MTREE_FILE} \
 		TMPPLIST=${TMPPLIST} \
-		DOCSDIR=${DOCSDIR} \
-		EXAMPLESDIR=${EXAMPLESDIR} \
-		PLIST_SUB='${PLIST_SUB:NPREFIX=*:NLOCALBASE=*:NOSREL=*:NLIB32DIR=*:NDOCSDIR=*:NEXAMPLESDIR=*:N*="* *"}'
+		SCRIPTSDIR=${SCRIPTSDIR} \
+		WITH_PKGNG=${WITH_PKGNG} \
+		PLIST_SUB_SED="${PLIST_SUB_SED}" \
+		PORT_OPTIONS="${PORT_OPTIONS}" \
+		PORTSDIR="${PORTSDIR}"
+.if defined(WITH_PKGNG)
+CO_ENV+=	PACKAGE_DEPENDS="${_LIB_RUN_DEPENDS:C,[^:]*:([^:]*):?.*,\1,:C,${PORTSDIR}/,,}" \
+		PKG_QUERY="${PKG_QUERY}"
+.else
+CO_ENV+=	PACKAGE_DEPENDS=${ACTUAL-PACKAGE-DEPENDS:Q} \
+		PKG_QUERY="${PKG_INFO}"
+.endif
+.if defined(NO_PREFIX_RMDIR)
+CO_ENV+=	NO_PREFIX_RMDIR=1
+.else
+CO_ENV+=	NO_PREFIX_RMDIR=0
+.endif
 
 .if !target(stage-dir)
 stage-dir:
@@ -74,10 +88,15 @@ makeplist: stage
 	@${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh makeplist
 .endif
 
+.if !target(check-plist)
+check-plist: stage
+	@${ECHO_MSG} "====> Checking for pkg-plist issues (check-plist)"
+	@${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh checkplist
+	@${ECHO_MSG} "===> No pkg-plist issues found (check-plist)"
+.endif
+
 .if !target(check-orphans)
-check-orphans: stage
-	@${ECHO_MSG} "====> Items missing from pkg-plist (check-orphans)"
-	@${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh orphans
+check-orphans: check-plist
 .endif
 
 .if !target(stage-qa)