diff options
| -rw-r--r-- | CHANGES | 16 | ||||
| -rw-r--r-- | Mk/Scripts/check-stagedir.sh | 260 | ||||
| -rw-r--r-- | Mk/Scripts/check_leftovers.sh | 2 | ||||
| -rw-r--r-- | Mk/Scripts/plist_sub_sed_sort.sh | 17 | ||||
| -rw-r--r-- | Mk/bsd.options.mk | 10 | ||||
| -rw-r--r-- | Mk/bsd.stage.mk | 31 |
6 files changed, 283 insertions, 53 deletions
@@ -10,6 +10,22 @@ in the release notes and/or placed into UPDATING. All ports committers are allowed to commit to this file. +20140419: +AUTHOR: bdrewery@FreeBSD.org + + check-orphans has been renamed to check-plist. It now checks: + A. Files in STAGEDIR that are missing from plist. + To make check-plist ignore a file *as an orphan* do one of the + following: + 1. Install it + 2. post-install: ${RM} ${STAGEDIR}file + 3. Put the file behind an OPTION with a PLIST_SUB: %%OPTION%%file + 4. Add to plist as a @comment + @comment file + @comment @dirrmtry dir + B. Files in plist missing from STAGEDIR + C. Files in plist which are owned by dependencies/MTREEs + 20140416: AUTHOR: bdrewery@FreeBSD.org diff --git a/Mk/Scripts/check-stagedir.sh b/Mk/Scripts/check-stagedir.sh index e2085718be27..0f7af97956c4 100644 --- a/Mk/Scripts/check-stagedir.sh +++ b/Mk/Scripts/check-stagedir.sh @@ -1,9 +1,19 @@ #!/bin/sh # ports/Mk/Scripts/check-stagedir.sh - called from ports/Mk/bsd.stage.mk # $FreeBSD$ +# +# MAINTAINER: portmgr@FreeBSD.org +# +# This script serves 2 purposes: +# 1. Generate a plist +# 2. Test a plist for issues: +# a. Files in STAGEDIR that are missing from plist +# b. Files in plist missing from STAGEDIR +# c. Files in plist which are owned by dependencies/MTREEs set -e export LC_ALL=C +ret=0 # lists an mtree file's contents, prefixed to dir. listmtree() { # mtreefile prefix @@ -16,15 +26,16 @@ listmtree() { # mtreefile prefix # obtain operating mode from command line makeplist=0 case "$1" in - orphans) ;; + checkplist) ;; makeplist) makeplist=1 ;; - *) echo >&2 "Usage: $0 {orphans|makelist}" ; exit 1 ;; + *) echo >&2 "Usage: $0 {checkplist|makelist}" ; exit 1 ;; esac # validate environment envfault= for i in STAGEDIR PREFIX LOCALBASE WRKDIR WRKSRC MTREE_FILE GNOME_MTREE_FILE \ - TMPPLIST DOCSDIR EXAMPLESDIR PLIST_SUB + TMPPLIST PLIST_SUB_SED SCRIPTSDIR PACKAGE_DEPENDS WITH_PKGNG PKG_QUERY \ + PORT_OPTIONS NO_PREFIX_RMDIR do if ! ( eval ": \${${i}?}" ) 2>/dev/null ; then envfault="${envfault}${envfault:+" "}${i}" @@ -43,8 +54,23 @@ set -u # files to the pipe and dirs to a separate file. if [ $makeplist = 0 ] ; then # check for orphans + echo "===> Checking for items in STAGEDIR missing from pkg-plist" + cwd=${PREFIX} while read line; do + # Handle deactivated OPTIONS. Treat "@comment file" as being in + # the plist so it does not show up as an orphan. PLIST_SUB uses + # a @comment to deactive files. XXX: It would be better to + # make all ports use @ignore instead of @comment. + comment= + if [ ${makeplist} -eq 0 -a -z "${line%%@comment *}" ]; then + line="${line#@comment }" + # Remove @comment so it can be parsed as a file, + # but later prepend it again to create a list of + # all files commented and uncommented. + comment="@comment " + fi + case $line in @dirrm*|'@unexec rmdir'*) line="$(printf %s "$line" \ @@ -55,25 +81,49 @@ if [ $makeplist = 0 ] ; then -e 's/@dirrm(try)?[[:space:]]+//' \ -e 's/[[:space:]]+$//')" case "$line" in - /*) echo >&3 "$line" ;; - *) echo >&3 "$cwd/$line" ;; + /*) echo >&3 "${comment}${line%/}" ;; + *) echo >&3 "${comment}${cwd}/${line%/}" ;; esac ;; - @info*) + # Handle [file] Keywords + @info\ *|@sample\ *) + set -- $line + shift + echo "${comment}${cwd}/$@" + ;; + # Handle [dirrmty] Keywords + @fc\ *|@fcfontsdir\ *|@fontsdir\ *) set -- $line shift - echo "$cwd/$@" + echo "${comment}@dirrmtry ${cwd}/$@" ;; + # order matters here - we must check @cwd first because # otherwise the @cwd* would also match it first, shadowing the # @cwd) line. @cwd|@cd) cwd=${PREFIX} ;; - @cwd*|@cd*) set -- $line ; cwd=$2 ;; + @cwd*|@cd*) + set -- $line + cwd=$2 + # Don't set cwd=/ as it causes // in plist and + # won't match later. + [ "${cwd}" = "/" ] && cwd= + ;; @*) ;; - /*) echo "$line" ;; - *) echo "$cwd/$line" ;; + /*) echo "${comment}${line}" ;; + *) echo "${comment}${cwd}/${line}" ;; esac - done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | sort >${WRKDIR}/.plist-files + done < ${TMPPLIST} 3>${WRKDIR}/.plist-dirs-unsorted | \ + sort >${WRKDIR}/.plist-files + unset TMPPLIST + # Create the -no-comments files and trim out @comment from the plists. + # This is used for various tests later. + sed -e '/^@comment/d' ${WRKDIR}/.plist-dirs-unsorted \ + >${WRKDIR}/.plist-dirs-unsorted-no-comments + sed -i '' -e 's/^@comment //' ${WRKDIR}/.plist-dirs-unsorted + sed -e '/^@comment/d' ${WRKDIR}/.plist-files \ + >${WRKDIR}/.plist-files-no-comments + sed -i '' -e 's/^@comment //' ${WRKDIR}/.plist-files else # generate plist - pretend the plist had been empty : >${WRKDIR}/.plist-dirs-unsorted @@ -84,55 +134,173 @@ fi ### PRODUCE MTREE FILE { listmtree /etc/mtree/BSD.root.dist "" - #listmtree /etc/mtree/BSD.usr.dist /usr + listmtree /etc/mtree/BSD.usr.dist /usr listmtree /etc/mtree/BSD.var.dist /var + # Use MTREE_FILE if specified and it doesn't already match LOCALBASE if [ -n "${MTREE_FILE}" ]; then - listmtree "${MTREE_FILE}" "${PREFIX}" + if [ "${PREFIX}" != "${LOCALBASE}" -o \ + "${MTREE_FILE}" != "${PORTSDIR}/Templates/BSD.local.dist" \ + ]; then + listmtree "${MTREE_FILE}" "${PREFIX}" + fi fi + listmtree "${PORTSDIR}/Templates/BSD.local.dist" "${LOCALBASE}" if [ -n "${GNOME_MTREE_FILE}" ]; then listmtree "${GNOME_MTREE_FILE}" "${PREFIX}" fi + unset MTREE_FILE GNOME_MTREE_FILE - a=${PREFIX} - while :; do - echo ${a} - a=${a%/*} - [ -z "${a}" ] && break - done -} > ${WRKDIR}/.mtree + # Add in PREFIX if this port wants it + if [ ${NO_PREFIX_RMDIR} -eq 0 ]; then + a=${PREFIX} + while :; do + echo ${a} + a=${a%/*} + [ -z "${a}" ] && break + done + fi +} >${WRKDIR}/.mtree -for i in $PLIST_SUB -do - echo $i -done | awk -F= '{sub(/^"/, "", $2); sub(/"$/, "", $2); print length($2), $1, $2 | "sort -nr" }' | while read l k v -do - if [ $l -gt 1 ] - then - echo "s,${v},%%${k}%%,g;" +### GATHER DIRS OWNED BY RUN-DEPENDS. WHY ARE WE SCREAMING? +: >${WRKDIR}/.run-depends-dirs +if [ -n "${WITH_PKGNG}" ]; then + echo "${PACKAGE_DEPENDS}" | xargs ${PKG_QUERY} "%D" | \ + sed -e 's,/$,,' | sort -u >>${WRKDIR}/.run-depends-dirs +else + # Evaluate ACTUAL-PACKAGE-DEPENDS + packagelist= + package_depends=$(eval ${PACKAGE_DEPENDS}) + if [ -n "${package_depends}" ]; then + # This ugly mess can go away with pkg_install EOL + awk_script=$(cat <<'EOF' + /Deinstall directory remove:/ {print $4} + /UNEXEC 'rmdir "[^"]*" 2>\/dev\/null \|\| true'/ { + gsub(/"%D\//, "\"", $0) + match($0, /"[^"]*"/) + dir=substr($0, RSTART+1, RLENGTH-2) + print dir + } +EOF +) + echo "${package_depends}" | while read line; do + ${PKG_QUERY} -f ${line%%:*} | \ + awk "${awk_script}" | \ + sed -e "/^[^/]/s,^,${LOCALBASE}/," + done | sort -u >>${WRKDIR}/.run-depends-dirs fi -done > ${WRKDIR}/.plist_sub +fi +unset PACKAGE_DEPENDS PKG_QUERY + +### HANDLE PORTDOCS/PORTEXAMPLES +sed_portdocsexamples="/%%DOCSDIR%%/s!^!%%PORTDOCS%%!g; /%%EXAMPLESDIR%%/s!^!%%PORTEXAMPLES%%!g;" +if [ ${makeplist} -eq 0 ]; then +# echo "=====> Using OPTIONS: ${PORT_OPTIONS}" | /usr/bin/fmt -w 79 | \ +# sed -e '2,$s/^/ /' + # Handle magical PORT* features + for option in DOCS EXAMPLES; do + want_option=0 + case " ${PORT_OPTIONS} " in + *\ ${option}\ *) want_option=1 ;; + esac + [ ${want_option} -eq 0 ] && \ + sed_portdocsexamples="${sed_portdocsexamples} /^%%PORT${option}%%/d;" + done + unset PORT_OPTIONS +fi -sed_plist_sub=`cat ${WRKDIR}/.plist_sub` +sed_plist_sub=$(echo "${PLIST_SUB_SED}" | /bin/sh ${SCRIPTSDIR}/plist_sub_sed_sort.sh) +unset PLIST_SUB_SED +sed_files="s!${PREFIX}/!!g; ${sed_plist_sub} ${sed_portdocsexamples} \ + /^share\/licenses/d;" + +sed_dirs="s!${PREFIX}/!!g; ${sed_plist_sub} s,^,@dirrmtry ,; \ + ${sed_portdocsexamples} \ + s!@dirrmtry \(/.*\)!@unexec rmdir \"\1\" >/dev/null 2>\&1 || :!; \ + /^@dirrmtry share\/licenses/d;" + +# If checking orphans, send all output to a temp file so whitelisting can be +# done +: >${WRKDIR}/.staged-plist ### HANDLE FILES -find ${STAGEDIR} -type f -o -type l | sort | sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files -comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files \ - | sed \ - -e "s,${DOCSDIR},%%PORTDOCS%%%%DOCSDIR%%,g" \ - -e "s,${EXAMPLESDIR},%%PORTEXAMPLES%%%%EXAMPLESDIR%%,g" \ - -e "s,${PREFIX}/,,g" \ - -e "${sed_plist_sub}" | grep -v "^share/licenses" || [ $? = 1 ] +find ${STAGEDIR} -type f -o -type l | sort | \ + sed -e "s,${STAGEDIR},," >${WRKDIR}/.staged-files +comm -13 ${WRKDIR}/.plist-files ${WRKDIR}/.staged-files | \ + sed -e "${sed_files}" \ + >>${WRKDIR}/.staged-plist || : ### HANDLE DIRS -cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree | sort -u >${WRKDIR}/.traced-dirs -find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | sort >${WRKDIR}/.staged-dirs +cat ${WRKDIR}/.plist-dirs-unsorted ${WRKDIR}/.mtree \ + ${WRKDIR}/.run-depends-dirs | sort -u >${WRKDIR}/.traced-dirs +find ${STAGEDIR} -type d | sed -e "s,^${STAGEDIR},,;/^$/d" | \ + sort >${WRKDIR}/.staged-dirs comm -13 ${WRKDIR}/.traced-dirs ${WRKDIR}/.staged-dirs \ - | sort -r | sed \ - -e 's,^,@dirrmtry ,' \ - -e "s,\(.*\)${DOCSDIR},%%PORTDOCS%%\1%%DOCSDIR%%,g" \ - -e "s,\(.*\)${EXAMPLESDIR},%%PORTEXAMPLES%%\1%%EXAMPLESDIR%%,g" \ - -e "s,${PREFIX}/,,g" \ - -e "${sed_plist_sub}" \ - -e 's,@dirrmtry \(/.*\),@unexec rmdir >/dev/null 2>\&1 \1 || :,' | grep -v "^@dirrmtry share/licenses" || [ $? = 1 ] + | sort -r | sed "${sed_dirs}" \ + >>${WRKDIR}/.staged-plist || : + +# If just making plist, show results and exit successfully. +if [ ${makeplist} -eq 1 ]; then + cat ${WRKDIR}/.staged-plist + exit 0 +fi + +# Handle whitelisting +while read path; do + case "${path}" in + *.bak) ;; + *.orig) ;; + #*/info/dir|info/dir) ;; + *) + # An orphan was found, return non-zero status + ret=1 + echo "Error: Orphaned: ${path}" >&2 + ;; + esac +done < ${WRKDIR}/.staged-plist + +sort -u ${WRKDIR}/.plist-dirs-unsorted-no-comments \ + >${WRKDIR}/.plist-dirs-sorted-no-comments + +# Anything listed in plist and in restricted-dirs is a failure. I.e., +# it's owned by a run-time dependency or one of the MTREEs. +echo "===> Checking for directories owned by dependencies or MTREEs" +cat ${WRKDIR}/.mtree ${WRKDIR}/.run-depends-dirs | sort -u \ + >${WRKDIR}/.restricted-dirs +: >${WRKDIR}/.invalid-plist-dependencies +comm -12 ${WRKDIR}/.plist-dirs-sorted-no-comments ${WRKDIR}/.restricted-dirs \ + | sort -r | sed "${sed_dirs}" \ + >>${WRKDIR}/.invalid-plist-dependencies || : +if [ -s "${WRKDIR}/.invalid-plist-dependencies" ]; then + ret=1 + while read line; do + echo "Error: Owned by dependency: ${line}" >&2 + done < ${WRKDIR}/.invalid-plist-dependencies +fi + +echo "===> Checking for items in pkg-plist which are not in STAGEDIR" +: >${WRKDIR}/.invalid-plist-missing +comm -23 ${WRKDIR}/.plist-files-no-comments ${WRKDIR}/.staged-files | \ + sed -e "${sed_files}" \ + >>${WRKDIR}/.invalid-plist-missing || : + +comm -23 ${WRKDIR}/.plist-dirs-sorted-no-comments ${WRKDIR}/.staged-dirs \ + | sort -r | sed "${sed_dirs}" \ + >>${WRKDIR}/.invalid-plist-missing || : +if [ -s "${WRKDIR}/.invalid-plist-missing" ]; then + ret=1 + while read line; do + echo "Error: Missing: ${line}" >&2 + done < ${WRKDIR}/.invalid-plist-missing +fi + +if [ ${ret} -ne 0 ]; then + echo "===> Error: Plist issues found." >&2 + if [ "${PREFIX}" != "${LOCALBASE}" ]; then + echo "===> Warning: Test was done with PREFIX != LOCALBASE" + echo "===> Warning: The port may not be properly installing into PREFIX" + fi +fi + +exit ${ret} diff --git a/Mk/Scripts/check_leftovers.sh b/Mk/Scripts/check_leftovers.sh index a85ac5d13dd4..48e704464c02 100644 --- a/Mk/Scripts/check_leftovers.sh +++ b/Mk/Scripts/check_leftovers.sh @@ -41,7 +41,7 @@ else LOCALBASE=$(make -C ${portdir} -VLOCALBASE) fi homedirs=$(awk -F: -v users=$(make -C ${portdir} -V USERS|sed -e 's, ,|,g;/^$/d') 'users && $1 ~ users {print $9}' ${PORTSDIR}/UIDs|sort -u|sed -e "s|/usr/local|${PREFIX}|") -plistsub_sed=$(make -C ${portdir} -VPLIST_SUB_SED) +plistsub_sed=$(make -C ${portdir} -VPLIST_SUB_SED | /bin/sh ${PORTSDIR}/Mk/Scripts/plist_sub_sed_sort.sh) tmpplist=$(make -C ${portdir} -VTMPPLIST) while read modtype path extra; do diff --git a/Mk/Scripts/plist_sub_sed_sort.sh b/Mk/Scripts/plist_sub_sed_sort.sh new file mode 100644 index 000000000000..aa58c73e883c --- /dev/null +++ b/Mk/Scripts/plist_sub_sed_sort.sh @@ -0,0 +1,17 @@ +#! /bin/sh +# $FreeBSD$ +# +# MAINTAINER: portmgr@FreeBSD.org +# +# PLIST_SUB_SED helper to sort by longest value first. + +exec awk '{ + while (match($0, /s![^!]*![^!]*!g;/)) { + sedp=substr($0, RSTART, RLENGTH) + $0=substr($0, RSTART+RLENGTH) + split(sedp, a, "!") + # Convert \. to . for sorting. + gsub(/\\./, ".", a[2]) + print length(a[2]), sedp + } +}' | sort -rn | awk '{print $2}' | paste -s -d ' ' - diff --git a/Mk/bsd.options.mk b/Mk/bsd.options.mk index f326f4938c99..5130a813babc 100644 --- a/Mk/bsd.options.mk +++ b/Mk/bsd.options.mk @@ -139,14 +139,24 @@ _OPTIONS_DEPENDS= PKG FETCH EXTRACT PATCH BUILD LIB RUN # Set the default values for the global options, as defined by portmgr .if !defined(NOPORTDOCS) PORT_OPTIONS+= DOCS +.else +OPTIONS_WARNINGS+= "NOPORTDOCS" +WITHOUT+= DOCS +OPTIONS_WARNINGS_UNSET+= DOCS .endif .if !defined(WITHOUT_NLS) PORT_OPTIONS+= NLS +.else +WITHOUT+= NLS .endif .if !defined(NOPORTEXAMPLES) PORT_OPTIONS+= EXAMPLES +.else +OPTIONS_WARNINGS+= "NOPORTEXAMPLES" +WITHOUT+= EXAMPLES +OPTIONS_WARNINGS_UNSET+= EXAMPLES .endif PORT_OPTIONS+= IPV6 diff --git a/Mk/bsd.stage.mk b/Mk/bsd.stage.mk index e643635556d0..da0f53146458 100644 --- a/Mk/bsd.stage.mk +++ b/Mk/bsd.stage.mk @@ -24,9 +24,23 @@ CO_ENV+= STAGEDIR=${STAGEDIR} \ MTREE_FILE=${MTREE_FILE} \ GNOME_MTREE_FILE=${GNOME_MTREE_FILE} \ TMPPLIST=${TMPPLIST} \ - DOCSDIR=${DOCSDIR} \ - EXAMPLESDIR=${EXAMPLESDIR} \ - PLIST_SUB='${PLIST_SUB:NPREFIX=*:NLOCALBASE=*:NOSREL=*:NLIB32DIR=*:NDOCSDIR=*:NEXAMPLESDIR=*:N*="* *"}' + SCRIPTSDIR=${SCRIPTSDIR} \ + WITH_PKGNG=${WITH_PKGNG} \ + PLIST_SUB_SED="${PLIST_SUB_SED}" \ + PORT_OPTIONS="${PORT_OPTIONS}" \ + PORTSDIR="${PORTSDIR}" +.if defined(WITH_PKGNG) +CO_ENV+= PACKAGE_DEPENDS="${_LIB_RUN_DEPENDS:C,[^:]*:([^:]*):?.*,\1,:C,${PORTSDIR}/,,}" \ + PKG_QUERY="${PKG_QUERY}" +.else +CO_ENV+= PACKAGE_DEPENDS=${ACTUAL-PACKAGE-DEPENDS:Q} \ + PKG_QUERY="${PKG_INFO}" +.endif +.if defined(NO_PREFIX_RMDIR) +CO_ENV+= NO_PREFIX_RMDIR=1 +.else +CO_ENV+= NO_PREFIX_RMDIR=0 +.endif .if !target(stage-dir) stage-dir: @@ -74,10 +88,15 @@ makeplist: stage @${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh makeplist .endif +.if !target(check-plist) +check-plist: stage + @${ECHO_MSG} "====> Checking for pkg-plist issues (check-plist)" + @${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh checkplist + @${ECHO_MSG} "===> No pkg-plist issues found (check-plist)" +.endif + .if !target(check-orphans) -check-orphans: stage - @${ECHO_MSG} "====> Items missing from pkg-plist (check-orphans)" - @${SETENV} ${CO_ENV} ${SH} ${SCRIPTSDIR}/check-stagedir.sh orphans +check-orphans: check-plist .endif .if !target(stage-qa) |