diff -u -r -N squid-4.0.23/acinclude/os-deps.m4 squid-4.0.24/acinclude/os-deps.m4 --- squid-4.0.23/acinclude/os-deps.m4 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/acinclude/os-deps.m4 2018-03-08 02:16:46.000000000 +1300 @@ -36,62 +36,6 @@ ]) dnl SQUID_CHECK_FUNC_STRNSTR -dnl check that va_copy is implemented and works -dnl sets squid_cv_func_va_copy and defines HAVE_VA_COPY -AC_DEFUN([SQUID_CHECK_FUNC_VACOPY],[ - -# check that the system provides a functional va_copy call - -AH_TEMPLATE(HAVE_VA_COPY, [The system implements a functional va_copy() ]) -AC_CACHE_CHECK(if va_copy is implemented, squid_cv_func_va_copy, - AC_RUN_IFELSE([AC_LANG_SOURCE([[ - #include - #include - int f (int i, ...) { - va_list args1, args2; - va_start (args1, i); - va_copy (args2, args1); - if (va_arg (args2, int) != 42 || va_arg (args1, int) != 42) - return 1; - va_end (args1); va_end (args2); - return 0; - } - int main(int argc, char **argv) { return f (0, 42); } - ]])],[squid_cv_func_va_copy="yes"],[squid_cv_func_va_copy="no"],[:]) -) -if test "$squid_cv_func_va_copy" = "yes" ; then - AC_DEFINE(HAVE_VA_COPY, 1) -fi - -]) dnl SQUID_CHECK_FUNC_VACOPY - -dnl same sa SQUID_CHECK_FUNC_VACOPY, but checks __va_copy -dnl sets squid_cv_func___va_copy, and defines HAVE___VA_COPY -AC_DEFUN([SQUID_CHECK_FUNC___VACOPY],[ - -AH_TEMPLATE(HAVE___VA_COPY,[Some systems have __va_copy instead of va_copy]) -AC_CACHE_CHECK(if __va_copy is implemented, squid_cv_func___va_copy, - AC_RUN_IFELSE([AC_LANG_SOURCE([[ - #include - #include - int f (int i, ...) { - va_list args1, args2; - va_start (args1, i); - __va_copy (args2, args1); - if (va_arg (args2, int) != 42 || va_arg (args1, int) != 42) - return 1; - va_end (args1); va_end (args2); - return 0; - } - int main(int argc, char **argv) { return f (0, 42); } - ]])],[squid_cv_func___va_copy="yes"],[squid_cv_func___va_copy="no"],[:]) -) -if test "$squid_cv_func___va_copy" = "yes" ; then - AC_DEFINE(HAVE___VA_COPY, 1) -fi -]) dnl SQUID_CHECK_FUNC___VACOPY - - dnl check that epoll actually works dnl sets squid_cv_epoll_works to "yes" or "no" AC_DEFUN([SQUID_CHECK_EPOLL],[ diff -u -r -N squid-4.0.23/cfgaux/config.guess squid-4.0.24/cfgaux/config.guess --- squid-4.0.23/cfgaux/config.guess 2018-01-20 02:30:51.000000000 +1300 +++ squid-4.0.24/cfgaux/config.guess 2018-03-08 02:24:39.000000000 +1300 @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2017 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2017-11-07' +timestamp='2018-02-24' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2017 Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -107,9 +107,9 @@ dummy=$tmp/dummy ; tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; + ,,) echo "int x;" > "$dummy.c" ; for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then + if ($c -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then CC_FOR_BUILD="$c"; break ; fi ; done ; @@ -132,14 +132,14 @@ UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown -case "${UNAME_SYSTEM}" in +case "$UNAME_SYSTEM" in Linux|GNU|GNU/*) # If the system lacks a compiler, then just pick glibc. # We could probably try harder. LIBC=gnu - eval $set_cc_for_build - cat <<-EOF > $dummy.c + eval "$set_cc_for_build" + cat <<-EOF > "$dummy.c" #include #if defined(__UCLIBC__) LIBC=uclibc @@ -149,13 +149,20 @@ LIBC=gnu #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC' | sed 's, ,,g'` + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi ;; esac # Note: order is significant - the case branches are not exclusive. -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, @@ -169,30 +176,30 @@ # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ - /sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || \ + "/sbin/$sysctl" 2>/dev/null || \ + "/usr/sbin/$sysctl" 2>/dev/null || \ echo unknown)` - case "${UNAME_MACHINE_ARCH}" in + case "$UNAME_MACHINE_ARCH" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; earmv*) - arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` - endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` - machine=${arch}${endian}-unknown + arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + machine="${arch}${endian}"-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; esac # The Operating System including object format, if it has switched # to ELF recently (or will in the future) and ABI. - case "${UNAME_MACHINE_ARCH}" in + case "$UNAME_MACHINE_ARCH" in earm*) os=netbsdelf ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build + eval "$set_cc_for_build" if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ELF__ then @@ -208,10 +215,10 @@ ;; esac # Determine ABI tags. - case "${UNAME_MACHINE_ARCH}" in + case "$UNAME_MACHINE_ARCH" in earm*) expr='s/^earmv[0-9]/-eabi/;s/eb$//' - abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` ;; esac # The OS release @@ -219,52 +226,55 @@ # thus, need a distinct triplet. However, they do not need # kernel version information, so it can be replaced with a # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in + case "$UNAME_VERSION" in Debian*) release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` + release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}${abi}" + echo "$machine-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-bitrig${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" exit ;; *:OpenBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" exit ;; *:LibertyBSD:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` - echo ${UNAME_MACHINE_ARCH}-unknown-libertybsd${UNAME_RELEASE} + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" exit ;; *:MidnightBSD:*:*) - echo ${UNAME_MACHINE}-unknown-midnightbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" exit ;; *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" exit ;; *:SolidBSD:*:*) - echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" exit ;; macppc:MirBSD:*:*) - echo powerpc-unknown-mirbsd${UNAME_RELEASE} + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" exit ;; *:Sortix:*:*) - echo ${UNAME_MACHINE}-unknown-sortix + echo "$UNAME_MACHINE"-unknown-sortix exit ;; *:Redox:*:*) - echo ${UNAME_MACHINE}-unknown-redox + echo "$UNAME_MACHINE"-unknown-redox exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -316,7 +326,7 @@ # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -325,10 +335,10 @@ echo m68k-unknown-sysv4 exit ;; *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos + echo "$UNAME_MACHINE"-unknown-amigaos exit ;; *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos + echo "$UNAME_MACHINE"-unknown-morphos exit ;; *:OS/390:*:*) echo i370-ibm-openedition @@ -340,7 +350,7 @@ echo powerpc-ibm-os400 exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} + echo arm-acorn-riscix"$UNAME_RELEASE" exit ;; arm*:riscos:*:*|arm*:RISCOS:*:*) echo arm-unknown-riscos @@ -367,19 +377,19 @@ sparc) echo sparc-icl-nx7; exit ;; esac ;; s390x:SunOS:*:*) - echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" exit ;; i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) - echo i386-pc-auroraux${UNAME_RELEASE} + echo i386-pc-auroraux"$UNAME_RELEASE" exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) - eval $set_cc_for_build + eval "$set_cc_for_build" SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. @@ -392,13 +402,13 @@ SUN_ARCH=x86_64 fi fi - echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo "$SUN_ARCH"-pc-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in @@ -407,25 +417,25 @@ ;; esac # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" exit ;; sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) - echo m68k-sun-sunos${UNAME_RELEASE} + echo m68k-sun-sunos"$UNAME_RELEASE" ;; sun4) - echo sparc-sun-sunos${UNAME_RELEASE} + echo sparc-sun-sunos"$UNAME_RELEASE" ;; esac exit ;; aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} + echo sparc-auspex-sunos"$UNAME_RELEASE" exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not @@ -436,44 +446,44 @@ # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} + echo m68k-atari-mint"$UNAME_RELEASE" exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} + echo m68k-milan-mint"$UNAME_RELEASE" exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} + echo m68k-hades-mint"$UNAME_RELEASE" exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} + echo m68k-unknown-mint"$UNAME_RELEASE" exit ;; m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} + echo m68k-apple-machten"$UNAME_RELEASE" exit ;; powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} + echo powerpc-apple-machten"$UNAME_RELEASE" exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 exit ;; RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} + echo mips-dec-ultrix"$UNAME_RELEASE" exit ;; VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} + echo vax-dec-ultrix"$UNAME_RELEASE" exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} + echo clipper-intergraph-clix"$UNAME_RELEASE" exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + eval "$set_cc_for_build" + sed 's/^ //' << EOF > "$dummy.c" #ifdef __cplusplus #include /* for printf() prototype */ int main (int argc, char *argv[]) { @@ -494,11 +504,11 @@ exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && - dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && - SYSTEM_NAME=`$dummy $dummyarg` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`"$dummy" "$dummyarg"` && { echo "$SYSTEM_NAME"; exit; } - echo mips-mips-riscos${UNAME_RELEASE} + echo mips-mips-riscos"$UNAME_RELEASE" exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax @@ -524,17 +534,17 @@ AViiON:dgux:*:*) # DG/UX returns AViiON for all architectures UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] + if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ + [ "$TARGET_BINARY_INTERFACE"x = x ] then - echo m88k-dg-dgux${UNAME_RELEASE} + echo m88k-dg-dgux"$UNAME_RELEASE" else - echo m88k-dg-dguxbcs${UNAME_RELEASE} + echo m88k-dg-dguxbcs"$UNAME_RELEASE" fi else - echo i586-dg-dgux${UNAME_RELEASE} + echo i586-dg-dgux"$UNAME_RELEASE" fi exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) @@ -551,7 +561,7 @@ echo m68k-tektronix-bsd exit ;; *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id @@ -563,14 +573,14 @@ if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + eval "$set_cc_for_build" + sed 's/^ //' << EOF > "$dummy.c" #include main() @@ -581,7 +591,7 @@ exit(0); } EOF - if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` then echo "$SYSTEM_NAME" else @@ -595,7 +605,7 @@ exit ;; *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 else IBM_ARCH=powerpc @@ -604,9 +614,9 @@ IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" exit ;; *:AIX:*:*) echo rs6000-ibm-aix @@ -615,7 +625,7 @@ echo romp-ibm-bsd4.4 exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx @@ -630,28 +640,28 @@ echo m68k-hp-bsd4.4 exit ;; 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + case "$UNAME_MACHINE" in 9000/31?) HP_ARCH=m68000 ;; 9000/[34]??) HP_ARCH=m68k ;; 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in + case "$sc_cpu_version" in 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in + case "$sc_kernel_bits" in 32) HP_ARCH=hppa2.0n ;; 64) HP_ARCH=hppa2.0w ;; '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + if [ "$HP_ARCH" = "" ]; then + eval "$set_cc_for_build" + sed 's/^ //' << EOF > "$dummy.c" #define _HPUX_SOURCE #include @@ -684,13 +694,13 @@ exit (0); } EOF - (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = hppa2.0w ] + if [ "$HP_ARCH" = hppa2.0w ] then - eval $set_cc_for_build + eval "$set_cc_for_build" # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler @@ -709,15 +719,15 @@ HP_ARCH=hppa64 fi fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" exit ;; ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux"$HPUX_REV" exit ;; 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + eval "$set_cc_for_build" + sed 's/^ //' << EOF > "$dummy.c" #include int main () @@ -742,7 +752,7 @@ exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 exit ;; @@ -763,9 +773,9 @@ exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk + echo "$UNAME_MACHINE"-unknown-osf1mk else - echo ${UNAME_MACHINE}-unknown-osf1 + echo "$UNAME_MACHINE"-unknown-osf1 fi exit ;; parisc*:Lites*:*:*) @@ -790,109 +800,109 @@ echo c4-convex-bsd exit ;; CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' exit ;; CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; *:UNICOS/mp:*:*) - echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" exit ;; sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} + echo sparc-unknown-bsdi"$UNAME_RELEASE" exit ;; *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" exit ;; *:FreeBSD:*:*) UNAME_PROCESSOR=`/usr/bin/uname -p` - case ${UNAME_PROCESSOR} in + case "$UNAME_PROCESSOR" in amd64) UNAME_PROCESSOR=x86_64 ;; i386) UNAME_PROCESSOR=i586 ;; esac - echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin + echo "$UNAME_MACHINE"-pc-cygwin exit ;; *:MINGW64*:*) - echo ${UNAME_MACHINE}-pc-mingw64 + echo "$UNAME_MACHINE"-pc-mingw64 exit ;; *:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 + echo "$UNAME_MACHINE"-pc-mingw32 exit ;; *:MSYS*:*) - echo ${UNAME_MACHINE}-pc-msys + echo "$UNAME_MACHINE"-pc-msys exit ;; i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 + echo "$UNAME_MACHINE"-pc-pw32 exit ;; *:Interix*:*) - case ${UNAME_MACHINE} in + case "$UNAME_MACHINE" in x86) - echo i586-pc-interix${UNAME_RELEASE} + echo i586-pc-interix"$UNAME_RELEASE" exit ;; authenticamd | genuineintel | EM64T) - echo x86_64-unknown-interix${UNAME_RELEASE} + echo x86_64-unknown-interix"$UNAME_RELEASE" exit ;; IA64) - echo ia64-unknown-interix${UNAME_RELEASE} + echo ia64-unknown-interix"$UNAME_RELEASE" exit ;; esac ;; i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin + echo "$UNAME_MACHINE"-pc-uwin exit ;; amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) echo x86_64-unknown-cygwin exit ;; prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" exit ;; *:GNU:*:*) # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-${LIBC}`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" exit ;; i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix + echo "$UNAME_MACHINE"-pc-minix exit ;; aarch64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; aarch64_be:Linux:*:*) UNAME_MACHINE=aarch64_be - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; alpha:Linux:*:*) case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in @@ -906,63 +916,63 @@ esac objdump --private-headers /bin/sh | grep -q ld.so.1 if test "$?" = 0 ; then LIBC=gnulibc1 ; fi - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arc:Linux:*:* | arceb:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; arm*:Linux:*:*) - eval $set_cc_for_build + eval "$set_cc_for_build" if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_EABI__ then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" else if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ | grep -q __ARM_PCS_VFP then - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi else - echo ${UNAME_MACHINE}-unknown-linux-${LIBC}eabihf + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf fi fi exit ;; avr32*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; cris:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; crisv32:Linux:*:*) - echo ${UNAME_MACHINE}-axis-linux-${LIBC} + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" exit ;; e2k:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; frv:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; hexagon:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:Linux:*:*) - echo ${UNAME_MACHINE}-pc-linux-${LIBC} + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" exit ;; ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; k1om:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m32r*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; mips:Linux:*:* | mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + eval "$set_cc_for_build" + sed 's/^ //' << EOF > "$dummy.c" #undef CPU #undef ${UNAME_MACHINE} #undef ${UNAME_MACHINE}el @@ -976,70 +986,74 @@ #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` - test x"${CPU}" != x && { echo "${CPU}-unknown-linux-${LIBC}"; exit; } + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" + test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } ;; mips64el:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; openrisc*:Linux:*:*) - echo or1k-unknown-linux-${LIBC} + echo or1k-unknown-linux-"$LIBC" exit ;; or32:Linux:*:* | or1k*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; padre:Linux:*:*) - echo sparc-unknown-linux-${LIBC} + echo sparc-unknown-linux-"$LIBC" exit ;; parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-${LIBC} + echo hppa64-unknown-linux-"$LIBC" exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-${LIBC} ;; - PA8*) echo hppa2.0-unknown-linux-${LIBC} ;; - *) echo hppa-unknown-linux-${LIBC} ;; + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; esac exit ;; ppc64:Linux:*:*) - echo powerpc64-unknown-linux-${LIBC} + echo powerpc64-unknown-linux-"$LIBC" exit ;; ppc:Linux:*:*) - echo powerpc-unknown-linux-${LIBC} + echo powerpc-unknown-linux-"$LIBC" exit ;; ppc64le:Linux:*:*) - echo powerpc64le-unknown-linux-${LIBC} + echo powerpc64le-unknown-linux-"$LIBC" exit ;; ppcle:Linux:*:*) - echo powerpcle-unknown-linux-${LIBC} + echo powerpcle-unknown-linux-"$LIBC" exit ;; riscv32:Linux:*:* | riscv64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; tile*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; vax:Linux:*:*) - echo ${UNAME_MACHINE}-dec-linux-${LIBC} + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-pc-linux-${LIBC} + if objdump -f /bin/sh | grep -q elf32-x86-64; then + echo "$UNAME_MACHINE"-pc-linux-"$LIBC"x32 + else + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" + fi exit ;; xtensa*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-${LIBC} + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. @@ -1053,34 +1067,34 @@ # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx + echo "$UNAME_MACHINE"-pc-os2-emx exit ;; i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop + echo "$UNAME_MACHINE"-unknown-stop exit ;; i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos + echo "$UNAME_MACHINE"-unknown-atheos exit ;; i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable + echo "$UNAME_MACHINE"-pc-syllable exit ;; i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} + echo i386-unknown-lynxos"$UNAME_RELEASE" exit ;; i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp + echo "$UNAME_MACHINE"-pc-msdosdjgpp exit ;; i*86:*:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" fi exit ;; i*86:*:5:[678]*) @@ -1090,12 +1104,12 @@ *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 @@ -1105,9 +1119,9 @@ && UNAME_MACHINE=i686 (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" else - echo ${UNAME_MACHINE}-pc-sysv32 + echo "$UNAME_MACHINE"-pc-sysv32 fi exit ;; pc:*:*:*) @@ -1127,9 +1141,9 @@ exit ;; i860:*:4.*:*) # i860-SVR4 if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 fi exit ;; mini*:CTIX:SYS*5:*) @@ -1149,9 +1163,9 @@ test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ && { echo i486-ncr-sysv4; exit; } ;; @@ -1160,28 +1174,28 @@ test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ - && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} + echo m68k-unknown-lynxos"$UNAME_RELEASE" exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 exit ;; TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} + echo sparc-unknown-lynxos"$UNAME_RELEASE" exit ;; rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} + echo rs6000-unknown-lynxos"$UNAME_RELEASE" exit ;; PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} + echo powerpc-unknown-lynxos"$UNAME_RELEASE" exit ;; SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} + echo mips-dde-sysv"$UNAME_RELEASE" exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 @@ -1192,7 +1206,7 @@ *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 + echo "$UNAME_MACHINE"-sni-sysv4 else echo ns32k-sni-sysv fi @@ -1212,23 +1226,23 @@ exit ;; i*86:VOS:*:*) # From Paul.Green@stratus.com. - echo ${UNAME_MACHINE}-stratus-vos + echo "$UNAME_MACHINE"-stratus-vos exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos exit ;; mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} + echo m68k-apple-aux"$UNAME_RELEASE" exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv"$UNAME_RELEASE" else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv"$UNAME_RELEASE" fi exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. @@ -1247,39 +1261,39 @@ echo x86_64-unknown-haiku exit ;; SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} + echo sx4-nec-superux"$UNAME_RELEASE" exit ;; SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} + echo sx5-nec-superux"$UNAME_RELEASE" exit ;; SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} + echo sx6-nec-superux"$UNAME_RELEASE" exit ;; SX-7:SUPER-UX:*:*) - echo sx7-nec-superux${UNAME_RELEASE} + echo sx7-nec-superux"$UNAME_RELEASE" exit ;; SX-8:SUPER-UX:*:*) - echo sx8-nec-superux${UNAME_RELEASE} + echo sx8-nec-superux"$UNAME_RELEASE" exit ;; SX-8R:SUPER-UX:*:*) - echo sx8r-nec-superux${UNAME_RELEASE} + echo sx8r-nec-superux"$UNAME_RELEASE" exit ;; SX-ACE:SUPER-UX:*:*) - echo sxace-nec-superux${UNAME_RELEASE} + echo sxace-nec-superux"$UNAME_RELEASE" exit ;; Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} + echo powerpc-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" exit ;; *:Darwin:*:*) UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown - eval $set_cc_for_build + eval "$set_cc_for_build" if test "$UNAME_PROCESSOR" = unknown ; then UNAME_PROCESSOR=powerpc fi - if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then + if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ @@ -1307,7 +1321,7 @@ # that Apple uses in portable devices. UNAME_PROCESSOR=x86_64 fi - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` @@ -1315,22 +1329,25 @@ UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" exit ;; *:QNX:*:4*) echo i386-pc-qnx exit ;; NEO-*:NONSTOP_KERNEL:*:*) - echo neo-tandem-nsk${UNAME_RELEASE} + echo neo-tandem-nsk"$UNAME_RELEASE" exit ;; NSE-*:NONSTOP_KERNEL:*:*) - echo nse-tandem-nsk${UNAME_RELEASE} + echo nse-tandem-nsk"$UNAME_RELEASE" exit ;; NSR-*:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} + echo nsr-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" exit ;; NSX-*:NONSTOP_KERNEL:*:*) - echo nsx-tandem-nsk${UNAME_RELEASE} + echo nsx-tandem-nsk"$UNAME_RELEASE" exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux @@ -1339,7 +1356,7 @@ echo bs2000-siemens-sysv exit ;; DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 @@ -1350,7 +1367,7 @@ else UNAME_MACHINE="$cputype" fi - echo ${UNAME_MACHINE}-unknown-plan9 + echo "$UNAME_MACHINE"-unknown-plan9 exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 @@ -1371,14 +1388,14 @@ echo pdp10-unknown-its exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} + echo mips-sei-seiux"$UNAME_RELEASE" exit ;; *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" exit ;; *:*VMS:*:*) UNAME_MACHINE=`(uname -p) 2>/dev/null` - case "${UNAME_MACHINE}" in + case "$UNAME_MACHINE" in A*) echo alpha-dec-vms ; exit ;; I*) echo ia64-dec-vms ; exit ;; V*) echo vax-dec-vms ; exit ;; @@ -1387,16 +1404,16 @@ echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` + echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" exit ;; i*86:rdos:*:*) - echo ${UNAME_MACHINE}-pc-rdos + echo "$UNAME_MACHINE"-pc-rdos exit ;; i*86:AROS:*:*) - echo ${UNAME_MACHINE}-pc-aros + echo "$UNAME_MACHINE"-pc-aros exit ;; x86_64:VMkernel:*:*) - echo ${UNAME_MACHINE}-unknown-esx + echo "$UNAME_MACHINE"-unknown-esx exit ;; amd64:Isilon\ OneFS:*:*) echo x86_64-unknown-onefs @@ -1405,7 +1422,7 @@ echo "$0: unable to guess system type" >&2 -case "${UNAME_MACHINE}:${UNAME_SYSTEM}" in +case "$UNAME_MACHINE:$UNAME_SYSTEM" in mips:Linux | mips64:Linux) # If we got here on MIPS GNU/Linux, output extra information. cat >&2 </dev/null` /usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" EOF exit 1 diff -u -r -N squid-4.0.23/cfgaux/config.sub squid-4.0.24/cfgaux/config.sub --- squid-4.0.23/cfgaux/config.sub 2018-01-20 02:30:51.000000000 +1300 +++ squid-4.0.24/cfgaux/config.sub 2018-03-08 02:24:39.000000000 +1300 @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2017 Free Software Foundation, Inc. +# Copyright 1992-2018 Free Software Foundation, Inc. -timestamp='2017-11-23' +timestamp='2018-02-22' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -67,7 +67,7 @@ version="\ GNU config.sub ($timestamp) -Copyright 1992-2017 Free Software Foundation, Inc. +Copyright 1992-2018 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -94,7 +94,7 @@ *local*) # First pass through any local machine types. - echo $1 + echo "$1" exit ;; * ) @@ -112,7 +112,7 @@ # Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). # Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` +maybe_os=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ @@ -120,16 +120,16 @@ kopensolaris*-gnu* | cloudabi*-eabi* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` + basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; android-linux) os=-linux-android - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + basic_machine=`echo "$1" | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown ;; *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` + basic_machine=`echo "$1" | sed 's/-[^-]*$//'` + if [ "$basic_machine" != "$1" ] + then os=`echo "$1" | sed 's/.*-/-/'` else os=; fi ;; esac @@ -178,44 +178,44 @@ ;; -sco6) os=-sco5v6 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco5) os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco4) os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco3.2.[4-9]*) os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco3.2v[4-9]*) # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco5v6*) # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -sco*) os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -isc) os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -clix*) basic_machine=clipper-intergraph ;; -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-pc/'` ;; -lynx*178) os=-lynxos178 @@ -227,7 +227,7 @@ os=-lynxos ;; -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + basic_machine=`echo "$1" | sed -e 's/86-.*/86-sequent/'` ;; -psos*) os=-psos @@ -296,7 +296,7 @@ | nios | nios2 | nios2eb | nios2el \ | ns16k | ns32k \ | open8 | or1k | or1knd | or32 \ - | pdp10 | pdp11 | pj | pjl \ + | pdp10 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ | pru \ | pyramid \ @@ -333,7 +333,7 @@ basic_machine=$basic_machine-unknown os=-none ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) ;; ms1) basic_machine=mt-unknown @@ -362,7 +362,7 @@ ;; # Object if more than one company name word. *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; # Recognize the basic CPU types with company name. @@ -457,7 +457,7 @@ # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. 386bsd) - basic_machine=i386-unknown + basic_machine=i386-pc os=-bsd ;; 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) @@ -491,7 +491,7 @@ basic_machine=x86_64-pc ;; amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; amdahl) basic_machine=580-amdahl @@ -536,7 +536,7 @@ os=-linux ;; blackfin-*) - basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; bluegene*) @@ -544,13 +544,13 @@ os=-cnk ;; c54x-*) - basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c55x-*) - basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c6x-*) - basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; c90) basic_machine=c90-cray @@ -648,7 +648,7 @@ os=$os"spe" ;; e500v[12]-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=$os"spe" ;; ebmon29k) @@ -740,9 +740,6 @@ hp9k8[0-9][0-9] | hp8[0-9][0-9]) basic_machine=hppa1.0-hp ;; - hppa-next) - os=-nextstep3 - ;; hppaosf) basic_machine=hppa1.1-hp os=-osf @@ -755,26 +752,26 @@ basic_machine=i370-ibm ;; i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv32 ;; i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv4 ;; i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-sysv ;; i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` os=-solaris2 ;; i386mach) basic_machine=i386-mach os=-mach ;; - i386-vsta | vsta) + vsta) basic_machine=i386-unknown os=-vsta ;; @@ -793,19 +790,16 @@ os=-sysv ;; leon-*|leon[3-9]-*) - basic_machine=sparc-`echo $basic_machine | sed 's/-.*//'` + basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` ;; m68knommu) basic_machine=m68k-unknown os=-linux ;; m68knommu-*) - basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; - m88k-omron*) - basic_machine=m88k-omron - ;; magnum | m3230) basic_machine=mips-mips os=-sysv @@ -837,10 +831,10 @@ os=-mint ;; mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` ;; mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown ;; monitor) basic_machine=m68k-rom68k @@ -859,7 +853,7 @@ os=-msdos ;; ms1-*) - basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` ;; msys) basic_machine=i686-pc @@ -946,6 +940,9 @@ nsr-tandem) basic_machine=nsr-tandem ;; + nsv-tandem) + basic_machine=nsv-tandem + ;; nsx-tandem) basic_machine=nsx-tandem ;; @@ -981,7 +978,7 @@ os=-linux ;; parisc-*) - basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` os=-linux ;; pbd) @@ -997,7 +994,7 @@ basic_machine=i386-pc ;; pc98-*) - basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc @@ -1012,16 +1009,16 @@ basic_machine=i786-pc ;; pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; pn) basic_machine=pn-gould @@ -1031,23 +1028,23 @@ ppc | ppcbe) basic_machine=powerpc-unknown ;; ppc-* | ppcbe-*) - basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64) basic_machine=powerpc64-unknown ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; ps2) basic_machine=i386-ibm @@ -1101,17 +1098,10 @@ sequent) basic_machine=i386-sequent ;; - sh) - basic_machine=sh-hitachi - os=-hms - ;; sh5el) basic_machine=sh5le-unknown ;; - sh64) - basic_machine=sh64-unknown - ;; - sparclite-wrs | simso-wrs) + simso-wrs) basic_machine=sparclite-wrs os=-vxworks ;; @@ -1130,7 +1120,7 @@ os=-sysv4 ;; strongarm-* | thumb-*) - basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` ;; sun2) basic_machine=m68000-sun @@ -1244,9 +1234,6 @@ basic_machine=a29k-wrs os=-vxworks ;; - wasm32) - basic_machine=wasm32-unknown - ;; w65*) basic_machine=w65-wdc os=-none @@ -1266,20 +1253,12 @@ basic_machine=xps100-honeywell ;; xscale-* | xscalee[bl]-*) - basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` ;; ymp) basic_machine=ymp-cray os=-unicos ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim - ;; - z80-*-coff) - basic_machine=z80-unknown - os=-sim - ;; none) basic_machine=none-none os=-none @@ -1308,10 +1287,6 @@ vax) basic_machine=vax-dec ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown - ;; pdp11) basic_machine=pdp11-dec ;; @@ -1321,9 +1296,6 @@ sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) basic_machine=sh-unknown ;; - sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) - basic_machine=sparc-sun - ;; cydra) basic_machine=cydra-cydrome ;; @@ -1343,7 +1315,7 @@ # Make sure to match an already-canonicalized machine name. ;; *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 exit 1 ;; esac @@ -1351,10 +1323,10 @@ # Here we canonicalize certain aliases for manufacturers. case $basic_machine in *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` ;; *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` ;; *) ;; @@ -1377,15 +1349,16 @@ -solaris) os=-solaris2 ;; - -svr4*) - os=-sysv4 - ;; -unixware*) os=-sysv4.2uw ;; -gnu/linux*) os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` ;; + # es1800 is here to avoid being matched by es* (a different OS) + -es1800*) + os=-ose + ;; # Now accept the basic system types. # The portable systems comes first. # Each alternative MUST end in a * to match a version number. @@ -1398,7 +1371,7 @@ | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -hiux* | -knetbsd* | -mirbsd* | -netbsd* \ | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ @@ -1409,14 +1382,15 @@ | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ + | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ + | -morphos* | -superux* | -rtmk* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ - | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*) + | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox* | -bme* \ + | -midnightbsd*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1433,12 +1407,12 @@ -nto*) os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ + -sim | -xray | -os68k* | -v88r* \ + | -windows* | -osx | -abug | -netware* | -os9* \ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) ;; -mac*) - os=`echo $os | sed -e 's|mac|macos|'` + os=`echo "$os" | sed -e 's|mac|macos|'` ;; -linux-dietlibc) os=-linux-dietlibc @@ -1447,10 +1421,10 @@ os=`echo $os | sed -e 's|linux|linux-gnu|'` ;; -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` ;; -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` ;; -opened*) os=-openedition @@ -1461,12 +1435,6 @@ -wince*) os=-wince ;; - -osfrose*) - os=-osfrose - ;; - -osf*) - os=-osf - ;; -utek*) os=-bsd ;; @@ -1513,7 +1481,7 @@ -oss*) os=-sysv3 ;; - -svr4) + -svr4*) os=-sysv4 ;; -svr3) @@ -1528,18 +1496,9 @@ -ose*) os=-ose ;; - -es1800*) - os=-ose - ;; - -xenix) - os=-xenix - ;; -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) os=-mint ;; - -aros*) - os=-aros - ;; -zvmoe) os=-zvmoe ;; @@ -1568,7 +1527,7 @@ *) # Get rid of the `-' at the beginning of $os. os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 exit 1 ;; esac @@ -1664,9 +1623,6 @@ *-be) os=-beos ;; - *-haiku) - os=-haiku - ;; *-ibm) os=-aix ;; @@ -1721,9 +1677,6 @@ i370-*) os=-mvs ;; - *-next) - os=-nextstep3 - ;; *-gould) os=-sysv ;; @@ -1833,11 +1786,11 @@ vendor=stratus ;; esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` ;; esac -echo $basic_machine$os +echo "$basic_machine$os" exit # Local variables: diff -u -r -N squid-4.0.23/ChangeLog squid-4.0.24/ChangeLog --- squid-4.0.23/ChangeLog 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/ChangeLog 2018-03-08 02:16:46.000000000 +1300 @@ -1,3 +1,14 @@ +Changes to squid-4.0.24 (07 Mar 2018): + + - Bug 4822: Build failure (-Wformat) where time_t is not long int + - Bug 4505: SMP caches sometimes do not purge entries + - TLS: GnuTLS implementation for listening ports and client connections + - TPROXY: Fix clientside_mark and client port logging + - Native FTP: Fix "Cannot assign requested address" with TPROXY + - SSL-Bump: Fix authentication with types other than Basic + - ... and many small compile and stability fixes + - ... and some documentation fixes + Changes to squid-4.0.23 (19 Jan 2018): - Bug 4715: security_file_certgen: Remove -g and -n options docs diff -u -r -N squid-4.0.23/configure squid-4.0.24/configure --- squid-4.0.23/configure 2018-01-20 02:30:55.000000000 +1300 +++ squid-4.0.24/configure 2018-03-08 02:24:46.000000000 +1300 @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Revision. # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.0.23. +# Generated by GNU Autoconf 2.69 for Squid Web Proxy 4.0.24. # # Report bugs to . # @@ -595,8 +595,8 @@ # Identity of this package. PACKAGE_NAME='Squid Web Proxy' PACKAGE_TARNAME='squid' -PACKAGE_VERSION='4.0.23' -PACKAGE_STRING='Squid Web Proxy 4.0.23' +PACKAGE_VERSION='4.0.24' +PACKAGE_STRING='Squid Web Proxy 4.0.24' PACKAGE_BUGREPORT='http://bugs.squid-cache.org/' PACKAGE_URL='' @@ -1647,7 +1647,7 @@ # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures Squid Web Proxy 4.0.23 to adapt to many kinds of systems. +\`configure' configures Squid Web Proxy 4.0.24 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1718,7 +1718,7 @@ if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of Squid Web Proxy 4.0.23:";; + short | recursive ) echo "Configuration of Squid Web Proxy 4.0.24:";; esac cat <<\_ACEOF @@ -2147,7 +2147,7 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -Squid Web Proxy configure 4.0.23 +Squid Web Proxy configure 4.0.24 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -3251,7 +3251,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by Squid Web Proxy $as_me 4.0.23, which was +It was created by Squid Web Proxy $as_me 4.0.24, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -4118,7 +4118,7 @@ # Define the identity of the package. PACKAGE='squid' - VERSION='4.0.23' + VERSION='4.0.24' cat >>confdefs.h <<_ACEOF @@ -4542,9 +4542,6 @@ - - - ## Copyright (C) 1996-2018 The Squid Software Foundation and contributors ## ## Squid software is distributed under GPLv2+ license and includes @@ -23380,12 +23377,12 @@ pkg_cv_LIBGNUTLS_CFLAGS="$LIBGNUTLS_CFLAGS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnutls >= 3.1.5\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gnutls >= 3.1.5") 2>&5 + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnutls >= 3.4.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnutls >= 3.4.0") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBGNUTLS_CFLAGS=`$PKG_CONFIG --cflags "gnutls >= 3.1.5" 2>/dev/null` + pkg_cv_LIBGNUTLS_CFLAGS=`$PKG_CONFIG --cflags "gnutls >= 3.4.0" 2>/dev/null` else pkg_failed=yes fi @@ -23396,12 +23393,12 @@ pkg_cv_LIBGNUTLS_LIBS="$LIBGNUTLS_LIBS" elif test -n "$PKG_CONFIG"; then if test -n "$PKG_CONFIG" && \ - { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnutls >= 3.1.5\""; } >&5 - ($PKG_CONFIG --exists --print-errors "gnutls >= 3.1.5") 2>&5 + { { $as_echo "$as_me:${as_lineno-$LINENO}: \$PKG_CONFIG --exists --print-errors \"gnutls >= 3.4.0\""; } >&5 + ($PKG_CONFIG --exists --print-errors "gnutls >= 3.4.0") 2>&5 ac_status=$? $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 test $ac_status = 0; }; then - pkg_cv_LIBGNUTLS_LIBS=`$PKG_CONFIG --libs "gnutls >= 3.1.5" 2>/dev/null` + pkg_cv_LIBGNUTLS_LIBS=`$PKG_CONFIG --libs "gnutls >= 3.4.0" 2>/dev/null` else pkg_failed=yes fi @@ -23421,9 +23418,9 @@ _pkg_short_errors_supported=no fi if test $_pkg_short_errors_supported = yes; then - LIBGNUTLS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "gnutls >= 3.1.5" 2>&1` + LIBGNUTLS_PKG_ERRORS=`$PKG_CONFIG --short-errors --print-errors "gnutls >= 3.4.0" 2>&1` else - LIBGNUTLS_PKG_ERRORS=`$PKG_CONFIG --print-errors "gnutls >= 3.1.5" 2>&1` + LIBGNUTLS_PKG_ERRORS=`$PKG_CONFIG --print-errors "gnutls >= 3.4.0" 2>&1` fi # Put the nasty error message in config.log where it belongs echo "$LIBGNUTLS_PKG_ERRORS" >&5 @@ -23431,10 +23428,10 @@ ## find the package without pkg-config ## check that the library is actually new enough. - ## by testing for a 3.1.5+ function which we use - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gnutls_certificate_verify_peers3 in -lgnutls" >&5 -$as_echo_n "checking for gnutls_certificate_verify_peers3 in -lgnutls... " >&6; } -if ${ac_cv_lib_gnutls_gnutls_certificate_verify_peers3+:} false; then : + ## by testing for a 3.4.0+ function which we use + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gnutls_pcert_export_x509 in -lgnutls" >&5 +$as_echo_n "checking for gnutls_pcert_export_x509 in -lgnutls... " >&6; } +if ${ac_cv_lib_gnutls_gnutls_pcert_export_x509+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -23448,27 +23445,27 @@ #ifdef __cplusplus extern "C" #endif -char gnutls_certificate_verify_peers3 (); +char gnutls_pcert_export_x509 (); int main () { -return gnutls_certificate_verify_peers3 (); +return gnutls_pcert_export_x509 (); ; return 0; } _ACEOF if ac_fn_cxx_try_link "$LINENO"; then : - ac_cv_lib_gnutls_gnutls_certificate_verify_peers3=yes + ac_cv_lib_gnutls_gnutls_pcert_export_x509=yes else - ac_cv_lib_gnutls_gnutls_certificate_verify_peers3=no + ac_cv_lib_gnutls_gnutls_pcert_export_x509=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" >&5 -$as_echo "$ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" >&6; } -if test "x$ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gnutls_gnutls_pcert_export_x509" >&5 +$as_echo "$ac_cv_lib_gnutls_gnutls_pcert_export_x509" >&6; } +if test "x$ac_cv_lib_gnutls_gnutls_pcert_export_x509" = xyes; then : LIBGNUTLS_LIBS="-lgnutls" fi @@ -23479,10 +23476,10 @@ ## find the package without pkg-config ## check that the library is actually new enough. - ## by testing for a 3.1.5+ function which we use - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gnutls_certificate_verify_peers3 in -lgnutls" >&5 -$as_echo_n "checking for gnutls_certificate_verify_peers3 in -lgnutls... " >&6; } -if ${ac_cv_lib_gnutls_gnutls_certificate_verify_peers3+:} false; then : + ## by testing for a 3.4.0+ function which we use + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for gnutls_pcert_export_x509 in -lgnutls" >&5 +$as_echo_n "checking for gnutls_pcert_export_x509 in -lgnutls... " >&6; } +if ${ac_cv_lib_gnutls_gnutls_pcert_export_x509+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -23496,27 +23493,27 @@ #ifdef __cplusplus extern "C" #endif -char gnutls_certificate_verify_peers3 (); +char gnutls_pcert_export_x509 (); int main () { -return gnutls_certificate_verify_peers3 (); +return gnutls_pcert_export_x509 (); ; return 0; } _ACEOF if ac_fn_cxx_try_link "$LINENO"; then : - ac_cv_lib_gnutls_gnutls_certificate_verify_peers3=yes + ac_cv_lib_gnutls_gnutls_pcert_export_x509=yes else - ac_cv_lib_gnutls_gnutls_certificate_verify_peers3=no + ac_cv_lib_gnutls_gnutls_pcert_export_x509=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext LIBS=$ac_check_lib_save_LIBS fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" >&5 -$as_echo "$ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" >&6; } -if test "x$ac_cv_lib_gnutls_gnutls_certificate_verify_peers3" = xyes; then : +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gnutls_gnutls_pcert_export_x509" >&5 +$as_echo "$ac_cv_lib_gnutls_gnutls_pcert_export_x509" >&6; } +if test "x$ac_cv_lib_gnutls_gnutls_pcert_export_x509" = xyes; then : LIBGNUTLS_LIBS="-lgnutls" fi @@ -23530,7 +23527,7 @@ CPPFLAGS="$CPPFLAGS $LIBGNUTLS_CFLAGS" fi - for ac_header in gnutls/gnutls.h gnutls/x509.h + for ac_header in gnutls/gnutls.h gnutls/x509.h gnutls/abstract.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_cxx_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" @@ -41442,101 +41439,6 @@ - -# check that the system provides a functional va_copy call - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if va_copy is implemented" >&5 -$as_echo_n "checking if va_copy is implemented... " >&6; } -if ${squid_cv_func_va_copy+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test "$cross_compiling" = yes; then : - : -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - #include - #include - int f (int i, ...) { - va_list args1, args2; - va_start (args1, i); - va_copy (args2, args1); - if (va_arg (args2, int) != 42 || va_arg (args1, int) != 42) - return 1; - va_end (args1); va_end (args2); - return 0; - } - int main(int argc, char **argv) { return f (0, 42); } - -_ACEOF -if ac_fn_cxx_try_run "$LINENO"; then : - squid_cv_func_va_copy="yes" -else - squid_cv_func_va_copy="no" -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $squid_cv_func_va_copy" >&5 -$as_echo "$squid_cv_func_va_copy" >&6; } -if test "$squid_cv_func_va_copy" = "yes" ; then - $as_echo "#define HAVE_VA_COPY 1" >>confdefs.h - -fi - - - - - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if __va_copy is implemented" >&5 -$as_echo_n "checking if __va_copy is implemented... " >&6; } -if ${squid_cv_func___va_copy+:} false; then : - $as_echo_n "(cached) " >&6 -else - if test "$cross_compiling" = yes; then : - : -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - - #include - #include - int f (int i, ...) { - va_list args1, args2; - va_start (args1, i); - __va_copy (args2, args1); - if (va_arg (args2, int) != 42 || va_arg (args1, int) != 42) - return 1; - va_end (args1); va_end (args2); - return 0; - } - int main(int argc, char **argv) { return f (0, 42); } - -_ACEOF -if ac_fn_cxx_try_run "$LINENO"; then : - squid_cv_func___va_copy="yes" -else - squid_cv_func___va_copy="no" -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - - -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $squid_cv_func___va_copy" >&5 -$as_echo "$squid_cv_func___va_copy" >&6; } -if test "$squid_cv_func___va_copy" = "yes" ; then - $as_echo "#define HAVE___VA_COPY 1" >>confdefs.h - -fi - - if test "x$enable_ipf_transparent" != "xno" ; then if test "x$squid_cv_broken_ipfilter_minor_t" = "x"; then @@ -43832,7 +43734,7 @@ # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by Squid Web Proxy $as_me 4.0.23, which was +This file was extended by Squid Web Proxy $as_me 4.0.24, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -43898,7 +43800,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -Squid Web Proxy config.status 4.0.23 +Squid Web Proxy config.status 4.0.24 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff -u -r -N squid-4.0.23/configure.ac squid-4.0.24/configure.ac --- squid-4.0.23/configure.ac 2018-01-20 02:30:55.000000000 +1300 +++ squid-4.0.24/configure.ac 2018-03-08 02:24:46.000000000 +1300 @@ -5,7 +5,7 @@ ## Please see the COPYING and CONTRIBUTORS files for details. ## -AC_INIT([Squid Web Proxy],[4.0.23],[http://bugs.squid-cache.org/],[squid]) +AC_INIT([Squid Web Proxy],[4.0.24],[http://bugs.squid-cache.org/],[squid]) AC_PREREQ(2.61) AC_CONFIG_HEADERS([include/autoconf.h]) AC_CONFIG_AUX_DIR(cfgaux) @@ -1268,15 +1268,15 @@ LIBS="$LIBS $LIBGNUTLS_PATH" # auto-detect using pkg-config - PKG_CHECK_MODULES([LIBGNUTLS],[gnutls >= 3.1.5],[ + PKG_CHECK_MODULES([LIBGNUTLS],[gnutls >= 3.4.0],[ CPPFLAGS="$CPPFLAGS $LIBGNUTLS_CFLAGS" ],[ ## find the package without pkg-config ## check that the library is actually new enough. - ## by testing for a 3.1.5+ function which we use - AC_CHECK_LIB(gnutls,gnutls_certificate_verify_peers3,[LIBGNUTLS_LIBS="-lgnutls"]) + ## by testing for a 3.4.0+ function which we use + AC_CHECK_LIB(gnutls,gnutls_pcert_export_x509,[LIBGNUTLS_LIBS="-lgnutls"]) ]) - AC_CHECK_HEADERS(gnutls/gnutls.h gnutls/x509.h) + AC_CHECK_HEADERS(gnutls/gnutls.h gnutls/x509.h gnutls/abstract.h) SQUID_STATE_ROLLBACK(squid_gnutls_state) #de-pollute LIBS @@ -3526,8 +3526,6 @@ ]) SQUID_CHECK_FUNC_STRNSTR -SQUID_CHECK_FUNC_VACOPY -SQUID_CHECK_FUNC___VACOPY dnl IP-Filter support requires ipf header files. These aren't dnl installed by default, so we need to check for them diff -u -r -N squid-4.0.23/doc/release-notes/release-4.html squid-4.0.24/doc/release-notes/release-4.html --- squid-4.0.23/doc/release-notes/release-4.html 2018-01-20 02:38:21.000000000 +1300 +++ squid-4.0.24/doc/release-notes/release-4.html 2018-03-08 02:33:23.000000000 +1300 @@ -2,10 +2,10 @@ - Squid 4.0.23 release notes + Squid 4.0.24 release notes -

Squid 4.0.23 release notes

+

Squid 4.0.24 release notes

Squid Developers

@@ -63,7 +63,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.0.23 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.0.24 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

@@ -251,10 +251,13 @@

2.8 Initial GnuTLS support

-

If all you need is a proxy that connects over TLS/SSL to a cache_peer -or accepts https:// URLs over clear-text and performs the necessary -upstream TLS connections. Then you now have the choice to build Squid with -GnuTLS instead of OpenSSL.

+

Squid can now be built to use GnuTLS in place of OpenSSL for the core +features of receiving TLS connections from clients and making TLS +connections to servers. The GnuTLS support is still very much experimental +and should be tested before use.

+ +

SSL-Bump and certificate generation features are not yet supported by +GnuTLS builds. Nor are many other less commonly used Squid TLS/SSL features.

squid.conf directives and configuration options which have undergone name changes from 'ssl' to 'tls' prefix in Squid-4 have GnuTLS support, unless @@ -298,6 +301,10 @@

New directive to limit the size of a table used for sharing information about collapsible entries among SMP workers.

+
force_request_body_continuation
+

New directive to control Squid behaviour on the client connection when +receiving an HTTP request with an Expect:100-continue header.

+
hopeless_kid_revival_delay

New directive to set a cool-down delay reviving a child process if the process is encountering frequent deaths.

@@ -306,6 +313,9 @@

New directive to set the action performed when encountering strange protocol requests at the beginning of an accepted TCP connection.

+
pconn_lifetime
+

New directive to limit the lifetime of persistent connections.

+
reply_header_add

New directive to add header fields to outgoing HTTP responses to the client.

@@ -347,6 +357,9 @@

Unused connections received in http_port or https_port or transactions terminated before reading[parsing] request headers logged with URI error:transaction-end-before-headers.

+

New option rotate= to control the number of log file rotations +to make when -k rotate command is received. Default is to +obey the logfile_rotate directive.

acl

New -m flag for note ACL to match substrings.

@@ -384,6 +397,14 @@

Replaced option sslcafile= with tls-cafile= which takes multiple entries.

+
deny_info
+

New format macro %O to expand the message= value supplied +by external ACL helpers.

+ +
ecap_service
+

New connection-encryption= option to determine ICAP service +effect on connections_encrypted ACL.

+
esi_parser

Removed custom parser option.

Changed default to auto-detect available parsers instead of custom.

@@ -422,6 +443,8 @@
icap_service

New scheme icaps:// to enable TLS/SSL connections to Secure ICAP servers on port 11344.

+

New connection-encryption= option to determine ICAP service +effect on connections_encrypted ACL.

New tls-cert= option to set TLS client certificate to use.

New tls-key= option to set TLS private key matching the client certificate used.

@@ -432,6 +455,8 @@

New tls-cipher= option to set a list of ciphers permitted.

New tls-cafile= option to set a file with additional CA certificate(s) to verify the server certificate.

+

New tls-capath= option to set a directory with additional CA +certificate(s) to verify the server certificate.

New tls-crlfile= option to set a file with a CRL to verify the server certificate.

New tls-default-ca option to use the system Trusted CAs to @@ -439,8 +464,13 @@

New tls-domain= option to verify the server certificate domain.

logformat
-

New code %ssl::<cert_errors to display server +

New quoting modifier to produce \-escaped output.

+

New code %ssl::<cert_errors to display server X.509 certificate errors.

+

New code %ssl::<cert_issuer to display Issuer field of +the received server X.509 certificate.

+

New code %ssl::<cert_subject to display Subject field of +the received server X.509 certificate.

New code %ssl::>negotiated_version to display negotiated TLS version of the client connection.

New code %ssl::<negotiated_version to display @@ -465,13 +495,15 @@

pid_filename

Default value now based on squid -n command line parameter.

+

This directive is no longer mandatory to edit for +multi-instance/tenant Squid installations.

refresh_pattern

Removed option ignore-auth. Its commonly desired behaviour is performed by default with correct HTTP/1.1 revalidation.

-

Removed ignore-must-revalidate. Other more HTTP compliant -directives (cache, store_miss) can be used to prevent objects from -caching.

+

Removed option ignore-must-revalidate. Other more HTTP compliant +directives (cache, store_miss) can be used to prevent +objects from caching.

sslcrtd_children

New parameter queue-size= to set the maximum number diff -u -r -N squid-4.0.23/include/autoconf.h.in squid-4.0.24/include/autoconf.h.in --- squid-4.0.23/include/autoconf.h.in 2018-01-20 02:30:48.000000000 +1300 +++ squid-4.0.24/include/autoconf.h.in 2018-03-08 02:24:38.000000000 +1300 @@ -320,6 +320,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_GNUMALLOC_H +/* Define to 1 if you have the header file. */ +#undef HAVE_GNUTLS_ABSTRACT_H + /* Define to 1 if you have the header file. */ #undef HAVE_GNUTLS_GNUTLS_H @@ -1170,9 +1173,6 @@ /* Define to 1 if you have the header file. */ #undef HAVE_VARARGS_H -/* The system implements a functional va_copy() */ -#undef HAVE_VA_COPY - /* Define to 1 if you have the `vfork' function. */ #undef HAVE_VFORK @@ -1224,9 +1224,6 @@ /* Define to 1 if you have the `__res_init' function. */ #undef HAVE___RES_INIT -/* Some systems have __va_copy instead of va_copy */ -#undef HAVE___VA_COPY - /* Enable ICAP client features in Squid */ #undef ICAP_CLIENT diff -u -r -N squid-4.0.23/include/version.h squid-4.0.24/include/version.h --- squid-4.0.23/include/version.h 2018-01-20 02:30:55.000000000 +1300 +++ squid-4.0.24/include/version.h 2018-03-08 02:24:46.000000000 +1300 @@ -7,7 +7,7 @@ */ #ifndef SQUID_RELEASE_TIME -#define SQUID_RELEASE_TIME 1516368643 +#define SQUID_RELEASE_TIME 1520429072 #endif /* diff -u -r -N squid-4.0.23/RELEASENOTES.html squid-4.0.24/RELEASENOTES.html --- squid-4.0.23/RELEASENOTES.html 2018-01-20 02:38:21.000000000 +1300 +++ squid-4.0.24/RELEASENOTES.html 2018-03-08 02:33:23.000000000 +1300 @@ -2,10 +2,10 @@ - Squid 4.0.23 release notes + Squid 4.0.24 release notes -

Squid 4.0.23 release notes

+

Squid 4.0.24 release notes

Squid Developers

@@ -63,7 +63,7 @@

1. Notice

-

The Squid Team are pleased to announce the release of Squid-4.0.23 for testing.

+

The Squid Team are pleased to announce the release of Squid-4.0.24 for testing.

This new release is available for download from http://www.squid-cache.org/Versions/v4/ or the mirrors.

@@ -251,10 +251,13 @@

2.8 Initial GnuTLS support

-

If all you need is a proxy that connects over TLS/SSL to a cache_peer -or accepts https:// URLs over clear-text and performs the necessary -upstream TLS connections. Then you now have the choice to build Squid with -GnuTLS instead of OpenSSL.

+

Squid can now be built to use GnuTLS in place of OpenSSL for the core +features of receiving TLS connections from clients and making TLS +connections to servers. The GnuTLS support is still very much experimental +and should be tested before use.

+ +

SSL-Bump and certificate generation features are not yet supported by +GnuTLS builds. Nor are many other less commonly used Squid TLS/SSL features.

squid.conf directives and configuration options which have undergone name changes from 'ssl' to 'tls' prefix in Squid-4 have GnuTLS support, unless @@ -298,6 +301,10 @@

New directive to limit the size of a table used for sharing information about collapsible entries among SMP workers.

+
force_request_body_continuation
+

New directive to control Squid behaviour on the client connection when +receiving an HTTP request with an Expect:100-continue header.

+
hopeless_kid_revival_delay

New directive to set a cool-down delay reviving a child process if the process is encountering frequent deaths.

@@ -306,6 +313,9 @@

New directive to set the action performed when encountering strange protocol requests at the beginning of an accepted TCP connection.

+
pconn_lifetime
+

New directive to limit the lifetime of persistent connections.

+
reply_header_add

New directive to add header fields to outgoing HTTP responses to the client.

@@ -347,6 +357,9 @@

Unused connections received in http_port or https_port or transactions terminated before reading[parsing] request headers logged with URI error:transaction-end-before-headers.

+

New option rotate= to control the number of log file rotations +to make when -k rotate command is received. Default is to +obey the logfile_rotate directive.

acl

New -m flag for note ACL to match substrings.

@@ -384,6 +397,14 @@

Replaced option sslcafile= with tls-cafile= which takes multiple entries.

+
deny_info
+

New format macro %O to expand the message= value supplied +by external ACL helpers.

+ +
ecap_service
+

New connection-encryption= option to determine ICAP service +effect on connections_encrypted ACL.

+
esi_parser

Removed custom parser option.

Changed default to auto-detect available parsers instead of custom.

@@ -422,6 +443,8 @@
icap_service

New scheme icaps:// to enable TLS/SSL connections to Secure ICAP servers on port 11344.

+

New connection-encryption= option to determine ICAP service +effect on connections_encrypted ACL.

New tls-cert= option to set TLS client certificate to use.

New tls-key= option to set TLS private key matching the client certificate used.

@@ -432,6 +455,8 @@

New tls-cipher= option to set a list of ciphers permitted.

New tls-cafile= option to set a file with additional CA certificate(s) to verify the server certificate.

+

New tls-capath= option to set a directory with additional CA +certificate(s) to verify the server certificate.

New tls-crlfile= option to set a file with a CRL to verify the server certificate.

New tls-default-ca option to use the system Trusted CAs to @@ -439,8 +464,13 @@

New tls-domain= option to verify the server certificate domain.

logformat
-

New code %ssl::<cert_errors to display server +

New quoting modifier to produce \-escaped output.

+

New code %ssl::<cert_errors to display server X.509 certificate errors.

+

New code %ssl::<cert_issuer to display Issuer field of +the received server X.509 certificate.

+

New code %ssl::<cert_subject to display Subject field of +the received server X.509 certificate.

New code %ssl::>negotiated_version to display negotiated TLS version of the client connection.

New code %ssl::<negotiated_version to display @@ -465,13 +495,15 @@

pid_filename

Default value now based on squid -n command line parameter.

+

This directive is no longer mandatory to edit for +multi-instance/tenant Squid installations.

refresh_pattern

Removed option ignore-auth. Its commonly desired behaviour is performed by default with correct HTTP/1.1 revalidation.

-

Removed ignore-must-revalidate. Other more HTTP compliant -directives (cache, store_miss) can be used to prevent objects from -caching.

+

Removed option ignore-must-revalidate. Other more HTTP compliant +directives (cache, store_miss) can be used to prevent +objects from caching.

sslcrtd_children

New parameter queue-size= to set the maximum number diff -u -r -N squid-4.0.23/src/acl/external/delayer/ext_delayer_acl.8 squid-4.0.24/src/acl/external/delayer/ext_delayer_acl.8 --- squid-4.0.23/src/acl/external/delayer/ext_delayer_acl.8 2018-01-20 02:38:22.000000000 +1300 +++ squid-4.0.24/src/acl/external/delayer/ext_delayer_acl.8 2018-03-08 02:33:24.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "EXT_DELAYER_ACL 8" -.TH EXT_DELAYER_ACL 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH EXT_DELAYER_ACL 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc squid-4.0.24/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc --- squid-4.0.23/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/acl/external/eDirectory_userip/ext_edirectory_userip_acl.cc 2018-03-08 02:16:46.000000000 +1300 @@ -53,7 +53,6 @@ #include #include #include -#include #include #include #include diff -u -r -N squid-4.0.23/src/acl/external/SQL_session/ext_sql_session_acl.8 squid-4.0.24/src/acl/external/SQL_session/ext_sql_session_acl.8 --- squid-4.0.23/src/acl/external/SQL_session/ext_sql_session_acl.8 2018-01-20 02:38:22.000000000 +1300 +++ squid-4.0.24/src/acl/external/SQL_session/ext_sql_session_acl.8 2018-03-08 02:33:24.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "EXT_SQL_SESSION_ACL 8" -.TH EXT_SQL_SESSION_ACL 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH EXT_SQL_SESSION_ACL 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/acl/external/time_quota/ext_time_quota_acl.cc squid-4.0.24/src/acl/external/time_quota/ext_time_quota_acl.cc --- squid-4.0.23/src/acl/external/time_quota/ext_time_quota_acl.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/acl/external/time_quota/ext_time_quota_acl.cc 2018-03-08 02:16:46.000000000 +1300 @@ -29,7 +29,6 @@ #include "squid.h" #include "helper/protocol_defines.h" -#include #include #include #include diff -u -r -N squid-4.0.23/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 squid-4.0.24/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 --- squid-4.0.23/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2018-01-20 02:38:23.000000000 +1300 +++ squid-4.0.24/src/acl/external/wbinfo_group/ext_wbinfo_group_acl.8 2018-03-08 02:33:24.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "EXT_WBINFO_GROUP_ACL 8" -.TH EXT_WBINFO_GROUP_ACL 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH EXT_WBINFO_GROUP_ACL 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/auth/AclProxyAuth.cc squid-4.0.24/src/auth/AclProxyAuth.cc --- squid-4.0.23/src/auth/AclProxyAuth.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/auth/AclProxyAuth.cc 2018-03-08 02:16:46.000000000 +1300 @@ -177,10 +177,10 @@ ACLProxyAuth::matchProxyAuth(ACLChecklist *cl) { ACLFilledChecklist *checklist = Filled(cl); - if (checklist->request->flags.sslBumped) - return 1; // AuthenticateAcl() already handled this bumped request - if (!authenticateUserAuthenticated(Filled(checklist)->auth_user_request)) { - return 0; + if (!checklist->request->flags.sslBumped) { + if (!authenticateUserAuthenticated(checklist->auth_user_request)) { + return 0; + } } /* check to see if we have matched the user-acl before */ int result = cacheMatchAcl(&checklist->auth_user_request->user()->proxy_match_cache, checklist); diff -u -r -N squid-4.0.23/src/auth/basic/DB/basic_db_auth.8 squid-4.0.24/src/auth/basic/DB/basic_db_auth.8 --- squid-4.0.23/src/auth/basic/DB/basic_db_auth.8 2018-01-20 02:38:23.000000000 +1300 +++ squid-4.0.24/src/auth/basic/DB/basic_db_auth.8 2018-03-08 02:33:25.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_DB_AUTH 8" -.TH BASIC_DB_AUTH 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH BASIC_DB_AUTH 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/auth/basic/POP3/basic_pop3_auth.8 squid-4.0.24/src/auth/basic/POP3/basic_pop3_auth.8 --- squid-4.0.23/src/auth/basic/POP3/basic_pop3_auth.8 2018-01-20 02:38:23.000000000 +1300 +++ squid-4.0.24/src/auth/basic/POP3/basic_pop3_auth.8 2018-03-08 02:33:25.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "BASIC_POP3_AUTH 8" -.TH BASIC_POP3_AUTH 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH BASIC_POP3_AUTH 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/cache_cf.cc squid-4.0.24/src/cache_cf.cc --- squid-4.0.23/src/cache_cf.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/cache_cf.cc 2018-03-08 02:16:46.000000000 +1300 @@ -915,8 +915,8 @@ for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) { if (!s->secure.encryptTransport) continue; - debugs(3, DBG_IMPORTANT, "Initializing " << AnyP::UriScheme(s->transport.protocol) << "_port " << s->s << " TLS context"); - s->secure.createSigningContexts(*s); + debugs(3, DBG_IMPORTANT, "Initializing " << AnyP::UriScheme(s->transport.protocol) << "_port " << s->s << " TLS contexts"); + s->secure.initServerContexts(*s); } // prevent infinite fetch loops in the request parser diff -u -r -N squid-4.0.23/src/cf.data.pre squid-4.0.24/src/cf.data.pre --- squid-4.0.23/src/cf.data.pre 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/cf.data.pre 2018-03-08 02:16:46.000000000 +1300 @@ -1481,7 +1481,7 @@ acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) -acl localhet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) acl localnet src fc00::/7 # RFC 4193 local private network range @@ -2122,12 +2122,33 @@ TLS / SSL Options: - cert= Path to SSL certificate (PEM format). + tls-cert= Path to file containing an X.509 certificate (PEM format) + to be used in the TLS handshake ServerHello. - key= Path to SSL private key file (PEM format) - if not specified, the certificate file is - assumed to be a combined certificate and - key file. + If this certificate is constrained by KeyUsage TLS + feature it must allow HTTP server usage, along with + any additional restrictions imposed by your choice + of options= settings. + + When OpenSSL is used this file may also contain a + chain of intermediate CA certificates to send in the + TLS handshake. + + When GnuTLS is used this option (and any paired + tls-key= option) may be repeated to load multiple + certificates for different domains. + + Also, when generate-host-certificates=on is configured + the first tls-cert= option must be a CA certificate + capable of signing the automatically generated + certificates. + + tls-key= Path to a file containing private key file (PEM format) + for the previous tls-cert= option. + + If tls-key= is not specified tls-cert= is assumed to + reference a PEM file containing both the certificate + and private key. cipher= Colon separated list of supported ciphers. NOTE: some ciphers such as EDH ciphers depend on @@ -2271,18 +2292,19 @@ DEFAULT: none LOC: HttpPortList DOC_START - Usage: [ip:]port [mode] cert=certificate.pem [options] + Usage: [ip:]port [mode] tls-cert=certificate.pem [options] The socket address where Squid will listen for client requests made over TLS or SSL connections. Commonly referred to as HTTPS. This is most useful for situations where you are running squid in - accelerator mode and you want to do the TLS work at the accelerator level. + accelerator mode and you want to do the TLS work at the accelerator + level. You may specify multiple socket addresses on multiple lines, each with their own certificate and/or options. - The TLS cert= option is mandatory on HTTPS ports. + The tls-cert= option is mandatory on HTTPS ports. See http_port for a list of modes and options. DOC_END @@ -2693,12 +2715,14 @@ disable Do not support https:// URLs. cert=/path/to/client/certificate - A client TLS certificate to use when connecting. + A client X.509 certificate to use when connecting. key=/path/to/client/private_key - The private TLS key corresponding to the cert= above. - If key= is not specified cert= is assumed to reference - a PEM file containing both the certificate and the key. + The private key corresponding to the cert= above. + + If key= is not specified cert= is assumed to + reference a PEM file containing both the certificate + and private key. cipher=... The list of valid TLS ciphers to use. @@ -3467,14 +3491,15 @@ tls Encrypt connections to this peer with TLS. sslcert=/path/to/ssl/certificate - A client SSL certificate to use when connecting to + A client X.509 certificate to use when connecting to this peer. sslkey=/path/to/ssl/key - The private SSL key corresponding to sslcert above. - If 'sslkey' is not specified 'sslcert' is assumed to - reference a combined file containing both the - certificate and the key. + The private key corresponding to sslcert above. + + If sslkey= is not specified sslcert= is assumed to + reference a PEM file containing both the certificate + and private key. sslcipher=... The list of valid SSL ciphers to use when connecting to this peer. @@ -4865,7 +4890,7 @@ Note, from Squid-3.1 this option is only a default for cache.log, that log can be rotated separately by using debug_options. - Note, from Squid-3.6 this option is only a default for access.log + Note, from Squid-4 this option is only a default for access.log recorded by stdio: module. Those logs can be rotated separately by using the rotate=N option on their access_log directive. @@ -8815,14 +8840,16 @@ These options are used for Secure ICAP (icaps://....) services only. tls-cert=/path/to/ssl/certificate - A client SSL certificate to use when connecting to - this icap server. + A client X.509 certificate to use when connecting to + this ICAP server. tls-key=/path/to/ssl/key - The private TLS/SSL key corresponding to sslcert above. - If 'tls-key' is not specified 'tls-cert' is assumed to - reference a combined PEM format file containing both the - certificate and the key. + The private key corresponding to the previous + tls-cert= option. + + If tls-key= is not specified tls-cert= is assumed to + reference a PEM file containing both the certificate + and private key. tls-cipher=... The list of valid TLS/SSL ciphers to use when connecting to this icap server. diff -u -r -N squid-4.0.23/src/clients/FtpClient.cc squid-4.0.24/src/clients/FtpClient.cc --- squid-4.0.23/src/clients/FtpClient.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/clients/FtpClient.cc 2018-03-08 02:16:46.000000000 +1300 @@ -763,6 +763,8 @@ conn->remote.port(data.port); conn->tos = ctrl.conn->tos; conn->nfmark = ctrl.conn->nfmark; + // Using non-local addresses in TPROXY mode requires appropriate socket option. + conn->flags |= ctrl.conn->flags & COMM_TRANSPARENT; debugs(9, 3, "connecting to " << conn->remote); diff -u -r -N squid-4.0.23/src/clients/FtpGateway.cc squid-4.0.24/src/clients/FtpGateway.cc --- squid-4.0.23/src/clients/FtpGateway.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/clients/FtpGateway.cc 2018-03-08 02:16:46.000000000 +1300 @@ -2641,14 +2641,10 @@ e->timestampsSet(); - if (flags.authenticated) { - /* - * Authenticated requests can't be cached. - */ - e->release(); - } else if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && !getCurrentOffset()) { - e->setPublicKey(); - } else { + // makePublic() if allowed/possible or release() otherwise + if (flags.authenticated || // authenticated requests can't be cached + getCurrentOffset() || + !e->makePublic()) { e->release(); } } diff -u -r -N squid-4.0.23/src/client_side.cc squid-4.0.24/src/client_side.cc --- squid-4.0.23/src/client_side.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/client_side.cc 2018-03-08 02:16:46.000000000 +1300 @@ -170,9 +170,6 @@ static void clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub); static IOACB httpAccept; -#if USE_OPENSSL -static IOACB httpsAccept; -#endif static CTCB clientLifetimeTimeout; #if USE_IDENT static IDCB clientIdentDone; @@ -2530,7 +2527,6 @@ AsyncJob::Start(srv); // usually async-calls readSomeData() } -#if USE_OPENSSL /// Create TLS connection structure and update fd_table static bool httpsCreate(const Comm::ConnectionPointer &conn, const Security::ContextPointer &ctx) @@ -2540,6 +2536,7 @@ return true; } + debugs(33, DBG_IMPORTANT, "ERROR: could not create TLS server context for " << conn); conn->close(); return false; } @@ -2551,50 +2548,74 @@ * \retval -1 on error */ static int -Squid_SSL_accept(ConnStateData *conn, PF *callback) +tlsAttemptHandshake(ConnStateData *conn, PF *callback) { + // TODO: maybe throw instead of returning -1 + // see https://github.com/squid-cache/squid/pull/81#discussion_r153053278 int fd = conn->clientConnection->fd; - auto ssl = fd_table[fd].ssl.get(); - int ret; + auto session = fd_table[fd].ssl.get(); errno = 0; - if ((ret = SSL_accept(ssl)) <= 0) { - const int xerrno = errno; - const int ssl_error = SSL_get_error(ssl, ret); - - switch (ssl_error) { - - case SSL_ERROR_WANT_READ: - Comm::SetSelect(fd, COMM_SELECT_READ, callback, (callback != NULL ? conn : NULL), 0); - return 0; - - case SSL_ERROR_WANT_WRITE: - Comm::SetSelect(fd, COMM_SELECT_WRITE, callback, (callback != NULL ? conn : NULL), 0); - return 0; - - case SSL_ERROR_SYSCALL: - if (ret == 0) { - debugs(83, 2, "Error negotiating SSL connection on FD " << fd << ": Aborted by client: " << ssl_error); - } else { - debugs(83, (xerrno == ECONNRESET) ? 1 : 2, "Error negotiating SSL connection on FD " << fd << ": " << - (xerrno == 0 ? Security::ErrorString(ssl_error) : xstrerr(xerrno))); - } - return -1; - case SSL_ERROR_ZERO_RETURN: - debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " << fd << ": Closed by client"); - return -1; +#if USE_OPENSSL + const auto ret = SSL_accept(session); + if (ret > 0) + return 1; + + const int xerrno = errno; + const auto ssl_error = SSL_get_error(session, ret); + + switch (ssl_error) { + + case SSL_ERROR_WANT_READ: + Comm::SetSelect(fd, COMM_SELECT_READ, callback, (callback ? conn : nullptr), 0); + return 0; + + case SSL_ERROR_WANT_WRITE: + Comm::SetSelect(fd, COMM_SELECT_WRITE, callback, (callback ? conn : nullptr), 0); + return 0; - default: - debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " << - fd << ": " << Security::ErrorString(ERR_get_error()) << - " (" << ssl_error << "/" << ret << ")"); - return -1; + case SSL_ERROR_SYSCALL: + if (ret == 0) { + debugs(83, 2, "Error negotiating SSL connection on FD " << fd << ": Aborted by client: " << ssl_error); + } else { + debugs(83, (xerrno == ECONNRESET) ? 1 : 2, "Error negotiating SSL connection on FD " << fd << ": " << + (xerrno == 0 ? Security::ErrorString(ssl_error) : xstrerr(xerrno))); } + break; - /* NOTREACHED */ + case SSL_ERROR_ZERO_RETURN: + debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " << fd << ": Closed by client"); + break; + + default: + debugs(83, DBG_IMPORTANT, "Error negotiating SSL connection on FD " << + fd << ": " << Security::ErrorString(ssl_error) << + " (" << ssl_error << "/" << ret << ")"); } - return 1; + +#elif USE_GNUTLS + + const auto x = gnutls_handshake(session); + if (x == GNUTLS_E_SUCCESS) + return 1; + + if (gnutls_error_is_fatal(x)) { + debugs(83, 2, "Error negotiating TLS on " << conn->clientConnection << ": Aborted by client: " << Security::ErrorString(x)); + + } else if (x == GNUTLS_E_INTERRUPTED || x == GNUTLS_E_AGAIN) { + const auto ioAction = (gnutls_record_get_direction(session)==0 ? COMM_SELECT_READ : COMM_SELECT_WRITE); + Comm::SetSelect(fd, ioAction, callback, (callback ? conn : nullptr), 0); + return 0; + } + +#else + // Performing TLS handshake should never be reachable without a TLS/SSL library. + (void)session; // avoid compiler and static analysis complaints + fatal("FATAL: HTTPS not supported by this Squid."); +#endif + + return -1; } /** negotiate an SSL connection */ @@ -2602,14 +2623,17 @@ clientNegotiateSSL(int fd, void *data) { ConnStateData *conn = (ConnStateData *)data; - int ret; - if ((ret = Squid_SSL_accept(conn, clientNegotiateSSL)) <= 0) { + + const int ret = tlsAttemptHandshake(conn, clientNegotiateSSL); + if (ret <= 0) { if (ret < 0) // An error conn->clientConnection->close(); return; } Security::SessionPointer session(fd_table[fd].ssl); + +#if USE_OPENSSL if (Security::SessionIsResumed(session)) { debugs(83, 2, "Session " << SSL_get_session(session.get()) << " reused on FD " << fd << " (" << fd_table[fd].ipaddr << @@ -2654,10 +2678,14 @@ " on FD " << fd << " (" << fd_table[fd].ipaddr << ":" << fd_table[fd].remote_port << ")"); } +#else + debugs(83, 2, "TLS session reuse not yet implemented."); +#endif // Connection established. Retrieve TLS connection parameters for logging. conn->clientConnection->tlsNegotiations()->retrieveNegotiatedInfo(session); +#if USE_OPENSSL X509 *client_cert = SSL_get_peer_certificate(session.get()); if (client_cert) { @@ -2669,8 +2697,11 @@ X509_free(client_cert); } else { - debugs(83, 5, "FD " << fd << " has no certificate."); + debugs(83, 5, "FD " << fd << " has no client certificate."); } +#else + debugs(83, 2, "Client certificate requesting not yet implemented."); +#endif conn->readSomeData(); } @@ -2696,6 +2727,7 @@ Comm::SetSelect(details->fd, COMM_SELECT_READ, clientNegotiateSSL, connState, 0); } +#if USE_OPENSSL /** * A callback function to use with the ACLFilledChecklist callback. */ @@ -2724,6 +2756,7 @@ if (!connState->fakeAConnectRequest("ssl-bump", connState->inBuf)) connState->clientConnection->close(); } +#endif /** handle a new HTTPS connection */ static void @@ -2757,6 +2790,7 @@ ConnStateData::postHttpsAccept() { if (port->flags.tunnelSslBumping) { +#if USE_OPENSSL debugs(33, 5, "accept transparent connection: " << clientConnection); if (!Config.accessList.ssl_bump) { @@ -2788,12 +2822,16 @@ acl_checklist->al->request = request; HTTPMSGLOCK(acl_checklist->al->request); acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, this); +#else + fatal("FATAL: SSL-Bump requires --with-openssl"); +#endif return; } else { httpsEstablish(this, port->secure.staticContext); } } +#if USE_OPENSSL void ConnStateData::sslCrtdHandleReplyWrapper(void *data, const Helper::Reply &reply) { @@ -2906,15 +2944,15 @@ assert(certProperties.signAlgorithm != Ssl::algSignEnd); if (certProperties.signAlgorithm == Ssl::algSignUntrusted) { - assert(port->secure.untrustedSigningCert); - certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCert.get()); - certProperties.signWithPkey.resetAndLock(port->secure.untrustedSignPkey.get()); + assert(port->secure.untrustedSigningCa.cert); + certProperties.signWithX509.resetAndLock(port->secure.untrustedSigningCa.cert.get()); + certProperties.signWithPkey.resetAndLock(port->secure.untrustedSigningCa.pkey.get()); } else { - assert(port->secure.signingCert.get()); - certProperties.signWithX509.resetAndLock(port->secure.signingCert.get()); + assert(port->secure.signingCa.cert.get()); + certProperties.signWithX509.resetAndLock(port->secure.signingCa.cert.get()); - if (port->secure.signPkey) - certProperties.signWithPkey.resetAndLock(port->secure.signPkey.get()); + if (port->secure.signingCa.pkey) + certProperties.signWithPkey.resetAndLock(port->secure.signingCa.pkey.get()); } signAlgorithm = certProperties.signAlgorithm; @@ -3022,7 +3060,7 @@ ConnStateData::getSslContextDone(Security::ContextPointer &ctx) { if (port->secure.generateHostCertificates && !ctx) { - debugs(33, 2, "Failed to generate TLS cotnext for " << sslConnectHostOrIp); + debugs(33, 2, "Failed to generate TLS context for " << sslConnectHostOrIp); } // If generated ssl context = NULL, try to use static ssl context. @@ -3253,7 +3291,7 @@ } // will call httpsPeeked() with certificate and connection, eventually - Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCert, port->secure.signPkey, port->secure)); + Security::ContextPointer unConfiguredCTX(Ssl::createSSLContext(port->secure.signingCa.cert, port->secure.signingCa.pkey, port->secure)); fd_table[clientConnection->fd].dynamicTlsContext = unConfiguredCTX; if (!httpsCreate(clientConnection, unConfiguredCTX)) @@ -3268,12 +3306,11 @@ bio->hold(true); // Here squid should have all of the client hello message so the - // Squid_SSL_accept should return 0; + // tlsAttemptHandshake() should return 0. // This block exist only to force openSSL parse client hello and detect // ERR_SECURE_ACCEPT_FAIL error, which should be checked and splice if required. - int ret = 0; - if ((ret = Squid_SSL_accept(this, NULL)) < 0) { - debugs(83, 2, "SSL_accept failed."); + if (tlsAttemptHandshake(this, nullptr) < 0) { + debugs(83, 2, "TLS handshake failed."); HttpRequest::Pointer request(http ? http->request : nullptr); if (!clientTunnelOnError(this, context, request, HttpRequestMethod(), ERR_SECURE_ACCEPT_FAIL)) clientConnection->close(); @@ -3496,12 +3533,12 @@ Ssl::TheGlobalContextStorage.addLocalStorage(s->s, s->secure.dynamicCertMemCacheSize); } } +#endif if (s->secure.encryptTransport && !s->secure.staticContext) { debugs(1, DBG_CRITICAL, "ERROR: Ignoring " << scheme << "_port " << s->s << " due to TLS context initialization failure."); continue; } -#endif // Fill out a Comm::Connection which IPC will open as a listener for us // then pass back when active so we can start a TcpAcceptor subscription. @@ -3521,7 +3558,6 @@ ListeningStartedDialer(&clientListenerConnectionOpened, s, Ipc::fdnHttpSocket, sub)); Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpSocket, listenCall); -#if USE_OPENSSL } else if (s->transport.protocol == AnyP::PROTO_HTTPS) { // setup the subscriptions such that new connections accepted by listenConn are handled by HTTPS RefCount subCall = commCbCall(5, 5, "httpsAccept", CommAcceptCbPtrFun(httpsAccept, CommAcceptCbParams(NULL))); @@ -3531,7 +3567,6 @@ ListeningStartedDialer(&clientListenerConnectionOpened, s, Ipc::fdnHttpsSocket, sub)); Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnHttpsSocket, listenCall); -#endif } HttpSockets[NHttpSockets] = -1; // set in clientListenerConnectionOpened diff -u -r -N squid-4.0.23/src/client_side.h squid-4.0.24/src/client_side.h --- squid-4.0.23/src/client_side.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/client_side.h 2018-03-08 02:16:46.000000000 +1300 @@ -216,10 +216,10 @@ /// The caller assumes responsibility for connection closure detection. void stopPinnedConnectionMonitoring(); -#if USE_OPENSSL /// the second part of old httpsAccept, waiting for future HttpsServer home void postHttpsAccept(); +#if USE_OPENSSL /// Initializes and starts a peek-and-splice negotiation with the SSL client void startPeekAndSplice(); diff -u -r -N squid-4.0.23/src/client_side_reply.cc squid-4.0.24/src/client_side_reply.cc --- squid-4.0.23/src/client_side_reply.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/client_side_reply.cc 2018-03-08 02:16:46.000000000 +1300 @@ -316,10 +316,8 @@ http->log_uri, http->request->flags, http->request->method); /* NOTE, don't call StoreEntry->lock(), storeCreateEntry() does it */ - if (collapsingAllowed) { + if (collapsingAllowed && Store::Root().allowCollapsing(entry, http->request->flags, http->request->method)) { debugs(88, 5, "allow other revalidation requests to collapse on " << *entry); - Store::Root().allowCollapsing(entry, http->request->flags, - http->request->method); collapsedRevalidation = crInitiator; } else { collapsedRevalidation = crNone; @@ -905,30 +903,16 @@ void purgeEntriesByUrl(HttpRequest * req, const char *url) { -#if USE_HTCP - bool get_or_head_sent = false; -#endif - for (HttpRequestMethod m(Http::METHOD_NONE); m != Http::METHOD_ENUM_END; ++m) { if (m.respMaybeCacheable()) { - if (StoreEntry *entry = storeGetPublic(url, m)) { - debugs(88, 5, "purging " << *entry << ' ' << m << ' ' << url); + const cache_key *key = storeKeyPublic(url, m); + debugs(88, 5, m << ' ' << url << ' ' << storeKeyText(key)); #if USE_HTCP - neighborsHtcpClear(entry, url, req, m, HTCP_CLR_INVALIDATION); - if (m == Http::METHOD_GET || m == Http::METHOD_HEAD) { - get_or_head_sent = true; - } + neighborsHtcpClear(nullptr, url, req, m, HTCP_CLR_INVALIDATION); #endif - entry->release(); - } + Store::Root().evictIfFound(key); } } - -#if USE_HTCP - if (!get_or_head_sent) { - neighborsHtcpClear(NULL, url, req, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_INVALIDATION); - } -#endif } void @@ -1059,7 +1043,7 @@ #if USE_HTCP neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); #endif - newEntry->release(); + newEntry->release(true); purgeStatus = Http::scOkay; } @@ -1075,7 +1059,7 @@ #if USE_HTCP neighborsHtcpClear(newEntry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); #endif - newEntry->release(); + newEntry->release(true); purgeStatus = Http::scOkay; } @@ -1091,7 +1075,7 @@ #if USE_HTCP neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_GET), HTCP_CLR_PURGE); #endif - entry->release(); + entry->release(true); purgeStatus = Http::scOkay; } @@ -1102,7 +1086,7 @@ #if USE_HTCP neighborsHtcpClear(entry, NULL, http->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_PURGE); #endif - entry->release(); + entry->release(true); purgeStatus = Http::scOkay; } } @@ -2296,7 +2280,7 @@ !reqFlags.needValidation && (m == Http::METHOD_GET || m == Http::METHOD_HEAD)) { // make the entry available for future requests now - Store::Root().allowCollapsing(e, reqFlags, m); + (void)Store::Root().allowCollapsing(e, reqFlags, m); } sc = storeClientListAdd(e, this); diff -u -r -N squid-4.0.23/src/CollapsedForwarding.cc squid-4.0.24/src/CollapsedForwarding.cc --- squid-4.0.23/src/CollapsedForwarding.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/CollapsedForwarding.cc 2018-03-08 02:16:46.000000000 +1300 @@ -53,27 +53,34 @@ } void -CollapsedForwarding::Broadcast(const StoreEntry &e) +CollapsedForwarding::Broadcast(const StoreEntry &e, const bool includingThisWorker) { - if (!queue.get()) - return; - - if (!e.mem_obj || e.mem_obj->xitTable.index < 0 || + if (!e.hasTransients() || !Store::Root().transientReaders(e)) { debugs(17, 7, "nobody reads " << e); return; } + debugs(17, 5, e); + Broadcast(e.mem_obj->xitTable.index, includingThisWorker); +} + +void +CollapsedForwarding::Broadcast(const sfileno index, const bool includingThisWorker) +{ + if (!queue.get()) + return; + CollapsedForwardingMsg msg; msg.sender = KidIdentifier; - msg.xitIndex = e.mem_obj->xitTable.index; + msg.xitIndex = index; - debugs(17, 5, e << " to " << Config.workers << "-1 workers"); + debugs(17, 7, "entry " << index << " to " << Config.workers << (includingThisWorker ? "" : "-1") << " workers"); // TODO: send only to workers who are waiting for data for (int workerId = 1; workerId <= Config.workers; ++workerId) { try { - if (workerId != KidIdentifier && queue->push(workerId, msg)) + if ((workerId != KidIdentifier || includingThisWorker) && queue->push(workerId, msg)) Notify(workerId); } catch (const Queue::Full &) { debugs(17, DBG_IMPORTANT, "ERROR: Collapsed forwarding " << diff -u -r -N squid-4.0.23/src/CollapsedForwarding.h squid-4.0.24/src/CollapsedForwarding.h --- squid-4.0.23/src/CollapsedForwarding.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/CollapsedForwarding.h 2018-03-08 02:16:46.000000000 +1300 @@ -13,6 +13,7 @@ #include "ipc/forward.h" #include "ipc/Queue.h" +#include "store/forward.h" #include @@ -26,7 +27,11 @@ static void Init(); /// notify other workers about changes in entry state (e.g., new data) - static void Broadcast(const StoreEntry &e); + static void Broadcast(const StoreEntry &e, const bool includingThisWorker = false); + + /// notify other workers about state changes in Transient entry at the given xitTable.index + /// use Broadcast(StoreEntry) variant if you have a StoreEntry object + static void Broadcast(const sfileno index, const bool includingThisWorker); /// kick worker with empty IPC queue static void Notify(const int workerId); diff -u -r -N squid-4.0.23/src/enums.h squid-4.0.24/src/enums.h --- squid-4.0.23/src/enums.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/enums.h 2018-03-08 02:16:46.000000000 +1300 @@ -47,9 +47,16 @@ STORE_PENDING } store_status_t; +/// StoreEntry relationship with a disk cache typedef enum { + /// StoreEntry is currently not associated with any disk store entry. + /// Does not guarantee (or preclude!) a matching disk store entry existence. SWAPOUT_NONE, + /// StoreEntry is being swapped out to the associated disk store entry. + /// Guarantees the disk store entry existence. SWAPOUT_WRITING, + /// StoreEntry is associated with a complete (i.e., fully swapped out) disk store entry. + /// Guarantees the disk store entry existence. SWAPOUT_DONE } swap_status_t; @@ -69,7 +76,7 @@ ENTRY_SPECIAL, ENTRY_REVALIDATE_ALWAYS, DELAY_SENDING, - RELEASE_REQUEST, + RELEASE_REQUEST, ///< prohibits making the key public REFRESH_REQUEST, ENTRY_REVALIDATE_STALE, ENTRY_DISPATCHED, diff -u -r -N squid-4.0.23/src/esi/Parser.cc squid-4.0.24/src/esi/Parser.cc --- squid-4.0.23/src/esi/Parser.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/esi/Parser.cc 2018-03-08 02:16:46.000000000 +1300 @@ -13,19 +13,27 @@ #include "fatal.h" char *ESIParser::Type = NULL; -ESIParser::Register *ESIParser::Parsers = NULL; ESIParser::Register *ESIParser::Parser = NULL; +std::list & +ESIParser::GetRegistry() +{ + static std::list parsers; + return parsers; +} + ESIParser::Pointer ESIParser::NewParser(ESIParserClient *aClient) { if (Parser == NULL) { - Parser = Parsers; + Parser = GetRegistry().front(); // if type name matters, use it if (strcasecmp(Type, "auto") != 0) { - while (Parser && strcasecmp(Parser->name, Type) != 0) - Parser = Parser->next; + for (auto *p : GetRegistry()) { + if (p && strcasecmp(p->name, Type) != 0) + Parser = p; + } } if (Parser == NULL) @@ -37,14 +45,11 @@ ESIParser::Register::Register(const char *_name, ESIParser::Pointer (*_newParser)(ESIParserClient *aClient)) : name(_name), newParser(_newParser) { - this->next = ESIParser::Parsers; - ESIParser::Parsers = this; + ESIParser::GetRegistry().emplace_back(this); } ESIParser::Register::~Register() { - // TODO: support random-order deregistration - assert(ESIParser::Parsers == this); - ESIParser::Parsers = next; + ESIParser::GetRegistry().remove(this); } diff -u -r -N squid-4.0.23/src/esi/Parser.h squid-4.0.24/src/esi/Parser.h --- squid-4.0.23/src/esi/Parser.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/esi/Parser.h 2018-03-08 02:16:46.000000000 +1300 @@ -9,6 +9,10 @@ #ifndef SQUID_ESIPARSER_H #define SQUID_ESIPARSER_H +#include "base/RefCount.h" + +#include + class ESIParserClient { public: @@ -19,8 +23,6 @@ virtual ~ESIParserClient() {}; }; -#include "base/RefCount.h" - class ESIParser : public RefCountable { public: @@ -45,9 +47,7 @@ private: static Register *Parser; - static Register *Parsers; - -public: + static std::list & GetRegistry(); }; class ESIParser::Register @@ -59,7 +59,6 @@ const char *name; ESIParser::Pointer (*newParser)(ESIParserClient *aClient); - Register * next; }; #define EsiParserDefinition(ThisClass) \ diff -u -r -N squid-4.0.23/src/fs/rock/RockSwapDir.cc squid-4.0.24/src/fs/rock/RockSwapDir.cc --- squid-4.0.23/src/fs/rock/RockSwapDir.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/rock/RockSwapDir.cc 2018-03-08 02:16:46.000000000 +1300 @@ -66,90 +66,68 @@ // create a brand new store entry and initialize it with stored basics StoreEntry *e = new StoreEntry(); + e->createMemObject(); anchorEntry(*e, filen, *slot); - - e->hashInsert(key); trackReferences(*e); - return e; - // the disk entry remains open for reading, protected from modifications } bool -Rock::SwapDir::anchorCollapsed(StoreEntry &collapsed, bool &inSync) +Rock::SwapDir::anchorToCache(StoreEntry &entry, bool &inSync) { if (!map || !theFile || !theFile->canRead()) return false; sfileno filen; const Ipc::StoreMapAnchor *const slot = map->openForReading( - reinterpret_cast(collapsed.key), filen); + reinterpret_cast(entry.key), filen); if (!slot) return false; - anchorEntry(collapsed, filen, *slot); - inSync = updateCollapsedWith(collapsed, *slot); + anchorEntry(entry, filen, *slot); + inSync = updateAnchoredWith(entry, *slot); return true; // even if inSync is false } bool -Rock::SwapDir::updateCollapsed(StoreEntry &collapsed) +Rock::SwapDir::updateAnchored(StoreEntry &entry) { if (!map || !theFile || !theFile->canRead()) return false; - if (collapsed.swap_filen < 0) // no longer using a disk cache - return true; - assert(collapsed.swap_dirn == index); + assert(entry.hasDisk(index)); - const Ipc::StoreMapAnchor &s = map->readableEntry(collapsed.swap_filen); - return updateCollapsedWith(collapsed, s); + const Ipc::StoreMapAnchor &s = map->readableEntry(entry.swap_filen); + return updateAnchoredWith(entry, s); } bool -Rock::SwapDir::updateCollapsedWith(StoreEntry &collapsed, const Ipc::StoreMapAnchor &anchor) +Rock::SwapDir::updateAnchoredWith(StoreEntry &entry, const Ipc::StoreMapAnchor &anchor) { - collapsed.swap_file_sz = anchor.basics.swap_file_sz; + entry.swap_file_sz = anchor.basics.swap_file_sz; return true; } void Rock::SwapDir::anchorEntry(StoreEntry &e, const sfileno filen, const Ipc::StoreMapAnchor &anchor) { - const Ipc::StoreMapAnchor::Basics &basics = anchor.basics; + anchor.exportInto(e); - e.swap_file_sz = basics.swap_file_sz; - e.lastref = basics.lastref; - e.timestamp = basics.timestamp; - e.expires = basics.expires; - e.lastModified(basics.lastmod); - e.refcount = basics.refcount; - e.flags = basics.flags; - - if (anchor.complete()) { - e.store_status = STORE_OK; - e.swap_status = SWAPOUT_DONE; - } else { - e.store_status = STORE_PENDING; - e.swap_status = SWAPOUT_WRITING; // even though another worker writes? - } + const bool complete = anchor.complete(); + e.store_status = complete ? STORE_OK : STORE_PENDING; + // SWAPOUT_WRITING: even though another worker writes? + e.attachToDisk(index, filen, complete ? SWAPOUT_DONE : SWAPOUT_WRITING); e.ping_status = PING_NONE; - EBIT_CLR(e.flags, RELEASE_REQUEST); - e.clearPrivate(); EBIT_SET(e.flags, ENTRY_VALIDATED); - - e.swap_dirn = index; - e.swap_filen = filen; } void Rock::SwapDir::disconnect(StoreEntry &e) { - assert(e.swap_dirn == index); - assert(e.swap_filen >= 0); - // cannot have SWAPOUT_NONE entry with swap_filen >= 0 - assert(e.swap_status != SWAPOUT_NONE); + assert(e.hasDisk(index)); + + ignoreReferences(e); // do not rely on e.swap_status here because there is an async delay // before it switches from SWAPOUT_WRITING to SWAPOUT_DONE. @@ -161,16 +139,12 @@ if (e.mem_obj && e.mem_obj->swapout.sio != NULL && dynamic_cast(*e.mem_obj->swapout.sio).writeableAnchor_) { map->abortWriting(e.swap_filen); - e.swap_dirn = -1; - e.swap_filen = -1; - e.swap_status = SWAPOUT_NONE; + e.detachFromDisk(); dynamic_cast(*e.mem_obj->swapout.sio).writeableAnchor_ = NULL; - Store::Root().transientsAbandon(e); // broadcasts after the change + Store::Root().stopSharing(e); // broadcasts after the change } else { map->closeForReading(e.swap_filen); - e.swap_dirn = -1; - e.swap_filen = -1; - e.swap_status = SWAPOUT_NONE; + e.detachFromDisk(); } } @@ -198,9 +172,16 @@ } void -Rock::SwapDir::swappedOut(const StoreEntry &) +Rock::SwapDir::finalizeSwapoutSuccess(const StoreEntry &) { - // stats are not stored but computed when needed + // nothing to do +} + +void +Rock::SwapDir::finalizeSwapoutFailure(StoreEntry &entry) +{ + debugs(47, 5, entry); + disconnect(entry); // calls abortWriting() to free the disk entry } int64_t @@ -771,7 +752,7 @@ return NULL; } - if (e.swap_filen < 0) { + if (!e.hasDisk()) { debugs(47,4, HERE << e); return NULL; } @@ -897,7 +878,7 @@ writeError(sio); sio.finishedWriting(errflag); - // and hope that Core will call disconnect() to close the map entry + // and wait for the finalizeSwapoutFailure() call to close the map entry } if (sio.touchingStoreEntry()) @@ -912,7 +893,7 @@ map->freeEntry(sio.swap_filen); // will mark as unusable, just in case if (sio.touchingStoreEntry()) - Store::Root().transientsAbandon(*sio.e); + Store::Root().stopSharing(*sio.e); // else noop: a fresh entry update error does not affect stale entry readers // All callers must also call IoState callback, to propagate the error. @@ -988,19 +969,24 @@ } void -Rock::SwapDir::unlink(StoreEntry &e) +Rock::SwapDir::evictIfFound(const cache_key *key) { - debugs(47, 5, HERE << e); - ignoreReferences(e); - map->freeEntry(e.swap_filen); - disconnect(e); + if (map) + map->freeEntryByKey(key); // may not be there } void -Rock::SwapDir::markForUnlink(StoreEntry &e) +Rock::SwapDir::evictCached(StoreEntry &e) { debugs(47, 5, e); - map->freeEntry(e.swap_filen); + if (e.hasDisk(index)) { + if (map->freeEntry(e.swap_filen)) + CollapsedForwarding::Broadcast(e); + if (!e.locked()) + disconnect(e); + } else if (const auto key = e.publicKey()) { + evictIfFound(key); + } } void @@ -1082,6 +1068,12 @@ return spacesPath.termedBuf(); } +bool +Rock::SwapDir::hasReadableEntry(const StoreEntry &e) const +{ + return map->hasReadableEntry(reinterpret_cast(e.key)); +} + namespace Rock { RunnerRegistrationEntry(SwapDirRr); diff -u -r -N squid-4.0.23/src/fs/rock/RockSwapDir.h squid-4.0.24/src/fs/rock/RockSwapDir.h --- squid-4.0.23/src/fs/rock/RockSwapDir.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/rock/RockSwapDir.h 2018-03-08 02:16:46.000000000 +1300 @@ -39,15 +39,18 @@ /* public ::SwapDir API */ virtual void reconfigure(); virtual StoreEntry *get(const cache_key *key); - virtual void markForUnlink(StoreEntry &e); + virtual void evictCached(StoreEntry &); + virtual void evictIfFound(const cache_key *); virtual void disconnect(StoreEntry &e); virtual uint64_t currentSize() const; virtual uint64_t currentCount() const; virtual bool doReportStat() const; - virtual void swappedOut(const StoreEntry &e); + virtual void finalizeSwapoutSuccess(const StoreEntry &); + virtual void finalizeSwapoutFailure(StoreEntry &); virtual void create(); virtual void parse(int index, char *path); virtual bool smpAware() const { return true; } + virtual bool hasReadableEntry(const StoreEntry &) const; // temporary path to the shared memory map of first slots of cached entries SBuf inodeMapPath() const; @@ -77,8 +80,8 @@ protected: /* Store API */ - virtual bool anchorCollapsed(StoreEntry &collapsed, bool &inSync); - virtual bool updateCollapsed(StoreEntry &collapsed); + virtual bool anchorToCache(StoreEntry &entry, bool &inSync); + virtual bool updateAnchored(StoreEntry &); /* protected ::SwapDir API */ virtual bool needsDiskStrand() const; @@ -94,7 +97,6 @@ virtual bool dereference(StoreEntry &e); virtual void updateHeaders(StoreEntry *e); virtual bool unlinkdUseful() const; - virtual void unlink(StoreEntry &e); virtual void statfs(StoreEntry &e) const; /* IORequestor API */ @@ -124,7 +126,7 @@ StoreIOState::Pointer createUpdateIO(const Ipc::StoreMapUpdate &update, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *); void anchorEntry(StoreEntry &e, const sfileno filen, const Ipc::StoreMapAnchor &anchor); - bool updateCollapsedWith(StoreEntry &collapsed, const Ipc::StoreMapAnchor &anchor); + bool updateAnchoredWith(StoreEntry &, const Ipc::StoreMapAnchor &); friend class Rebuild; friend class IoState; diff -u -r -N squid-4.0.23/src/fs/ufs/RebuildState.cc squid-4.0.24/src/fs/ufs/RebuildState.cc --- squid-4.0.23/src/fs/ufs/RebuildState.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/ufs/RebuildState.cc 2018-03-08 02:16:46.000000000 +1300 @@ -40,7 +40,6 @@ fn(0), entry(NULL), td(NULL), - e(NULL), fromLog(true), _done(false), cbdata(NULL) @@ -108,8 +107,6 @@ void Fs::Ufs::RebuildState::rebuildStep() { - currentEntry(NULL); - // Balance our desire to maximize the number of entries processed at once // (and, hence, minimize overheads and total rebuild time) with a // requirement to also process Coordinator events, disk I/Os, etc. @@ -208,34 +205,65 @@ return; } - if (!storeRebuildKeepEntry(tmpe, key, counts)) - return; - - ++counts.objcount; - // tmpe.dump(5); - currentEntry(sd->addDiskRestore(key, - filn, - tmpe.swap_file_sz, - tmpe.expires, - tmpe.timestamp, - tmpe.lastref, - tmpe.lastModified(), - tmpe.refcount, /* refcount */ - tmpe.flags, /* flags */ - (int) flags.clean)); - storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); + addIfFresh(key, + filn, + tmpe.swap_file_sz, + tmpe.expires, + tmpe.timestamp, + tmpe.lastref, + tmpe.lastModified(), + tmpe.refcount, + tmpe.flags); } -StoreEntry * -Fs::Ufs::RebuildState::currentEntry() const +/// if the loaded entry metadata is still relevant, indexes the entry +void +Fs::Ufs::RebuildState::addIfFresh(const cache_key *key, + sfileno file_number, + uint64_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + uint32_t refcount, + uint16_t newFlags) { - return e; -} + if (!evictStaleAndContinue(key, lastref, counts.dupcount)) + return; -void -Fs::Ufs::RebuildState::currentEntry(StoreEntry *newValue) + ++counts.objcount; + const auto addedEntry = sd->addDiskRestore(key, + file_number, + swap_file_sz, + expires, + timestamp, + lastref, + lastmod, + refcount, + newFlags, + 0 /* XXX: unused */); + storeDirSwapLog(addedEntry, SWAP_LOG_ADD); +} + +/// Evicts a matching entry if it was last touched before caller's maxRef. +/// \returns false only if the matching entry was touched at or after maxRef, +/// indicating that the caller has supplied outdated maxRef. +bool +Fs::Ufs::RebuildState::evictStaleAndContinue(const cache_key *candidateKey, const time_t maxRef, int &staleCount) { - e = newValue; + if (auto *indexedEntry = Store::Root().peek(candidateKey)) { + + if (indexedEntry->lastref >= maxRef) { + indexedEntry->abandon("Fs::Ufs::RebuildState::evictStaleAndContinue"); + ++counts.clashcount; + return false; + } + + ++staleCount; + indexedEntry->release(true); // evict previously indexedEntry + } + + return true; } /// process one swap log entry @@ -278,20 +306,8 @@ if (swapData.op == SWAP_LOG_ADD) { (void) 0; } else if (swapData.op == SWAP_LOG_DEL) { - /* Delete unless we already have a newer copy anywhere in any store */ - /* this needs to become - * 1) unpack url - * 2) make synthetic request with headers ?? or otherwise search - * for a matching object in the store - * TODO FIXME change to new async api - */ - currentEntry (Store::Root().get(swapData.key)); - - if (currentEntry() != NULL && swapData.lastref >= e->lastref) { - undoAdd(); - --counts.objcount; - ++counts.cancelcount; - } + // remove any older or same-age entry; +1 covers same-age entries + (void)evictStaleAndContinue(swapData.key, swapData.lastref+1, counts.cancelcount); return; } else { const double @@ -317,123 +333,23 @@ return; } - /* this needs to become - * 1) unpack url - * 2) make synthetic request with headers ?? or otherwise search - * for a matching object in the store - * TODO FIXME change to new async api - */ - currentEntry (Store::Root().get(swapData.key)); - - int used; /* is swapfile already in use? */ - - used = sd->mapBitTest(swapData.swap_filen); - - /* If this URL already exists in the cache, does the swap log - * appear to have a newer entry? Compare 'lastref' from the - * swap log to e->lastref. */ - /* is the log entry newer than current entry? */ - int disk_entry_newer = currentEntry() ? (swapData.lastref > currentEntry()->lastref ? 1 : 0) : 0; - - if (used && !disk_entry_newer) { - /* log entry is old, ignore it */ - ++counts.clashcount; - return; - } else if (used && currentEntry() && currentEntry()->swap_filen == swapData.swap_filen && currentEntry()->swap_dirn == sd->index) { - /* swapfile taken, same URL, newer, update meta */ - - if (currentEntry()->store_status == STORE_OK) { - currentEntry()->lastref = swapData.timestamp; - currentEntry()->timestamp = swapData.timestamp; - currentEntry()->expires = swapData.expires; - currentEntry()->lastModified(swapData.lastmod); - currentEntry()->flags = swapData.flags; - currentEntry()->refcount += swapData.refcount; - sd->dereference(*currentEntry()); - } else { - debug_trap("commonUfsDirRebuildFromSwapLog: bad condition"); - debugs(47, DBG_IMPORTANT, HERE << "bad condition"); - } - return; - } else if (used) { - /* swapfile in use, not by this URL, log entry is newer */ - /* This is sorta bad: the log entry should NOT be newer at this - * point. If the log is dirty, the filesize check should have - * caught this. If the log is clean, there should never be a - * newer entry. */ - debugs(47, DBG_IMPORTANT, "WARNING: newer swaplog entry for dirno " << - sd->index << ", fileno "<< std::setfill('0') << std::hex << - std::uppercase << std::setw(8) << swapData.swap_filen); - - /* I'm tempted to remove the swapfile here just to be safe, - * but there is a bad race condition in the NOVM version if - * the swapfile has recently been opened for writing, but - * not yet opened for reading. Because we can't map - * swapfiles back to StoreEntrys, we don't know the state - * of the entry using that file. */ - /* We'll assume the existing entry is valid, probably because - * were in a slow rebuild and the the swap file number got taken - * and the validation procedure hasn't run. */ - assert(flags.need_to_validate); + if (sd->mapBitTest(swapData.swap_filen)) { + // While we were scanning the logs, some (unrelated) entry was added to + // our disk using our logged swap_filen. We could change our swap_filen + // and move the store file, but there is no Store API to do that (TODO). ++counts.clashcount; return; - } else if (currentEntry() && !disk_entry_newer) { - /* key already exists, current entry is newer */ - /* keep old, ignore new */ - ++counts.dupcount; - return; - } else if (currentEntry()) { - /* key already exists, this swapfile not being used */ - /* junk old, load new */ - undoAdd(); - --counts.objcount; - ++counts.dupcount; - } else { - /* URL doesnt exist, swapfile not in use */ - /* load new */ - (void) 0; - } - - ++counts.objcount; - - currentEntry(sd->addDiskRestore(swapData.key, - swapData.swap_filen, - swapData.swap_file_sz, - swapData.expires, - swapData.timestamp, - swapData.lastref, - swapData.lastmod, - swapData.refcount, - swapData.flags, - (int) flags.clean)); - - storeDirSwapLog(currentEntry(), SWAP_LOG_ADD); -} - -/// undo the effects of adding an entry in rebuildFromSwapLog() -void -Fs::Ufs::RebuildState::undoAdd() -{ - StoreEntry *added = currentEntry(); - assert(added); - currentEntry(NULL); - - // TODO: Why bother with these two if we are going to release?! - added->expireNow(); - added->releaseRequest(); - - if (added->swap_filen > -1) { - SwapDir *someDir = INDEXSD(added->swap_dirn); - assert(someDir); - if (UFSSwapDir *ufsDir = dynamic_cast(someDir)) - ufsDir->undoAddDiskRestore(added); - // else the entry was loaded from and/or is currently in a non-UFS dir - // Thus, there is no use in preserving its disk file (the only purpose - // of undoAddDiskRestore!), even if we could. Instead, we release the - // the entry and [eventually] unlink its disk file or free its slot. } - added->release(); + addIfFresh(swapData.key, + swapData.swap_filen, + swapData.swap_file_sz, + swapData.expires, + swapData.timestamp, + swapData.lastref, + swapData.lastmod, + swapData.refcount, + swapData.flags); } int @@ -559,9 +475,3 @@ return _done; } -StoreEntry * -Fs::Ufs::RebuildState::currentItem() -{ - return currentEntry(); -} - diff -u -r -N squid-4.0.23/src/fs/ufs/RebuildState.h squid-4.0.24/src/fs/ufs/RebuildState.h --- squid-4.0.23/src/fs/ufs/RebuildState.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/ufs/RebuildState.h 2018-03-08 02:16:46.000000000 +1300 @@ -33,7 +33,6 @@ virtual bool error() const; virtual bool isDone() const; - virtual StoreEntry *currentItem(); RefCount sd; int n_read; @@ -63,11 +62,17 @@ void rebuildFromDirectory(); void rebuildFromSwapLog(); void rebuildStep(); - void undoAdd(); + void addIfFresh(const cache_key *key, + sfileno file_number, + uint64_t swap_file_sz, + time_t expires, + time_t timestamp, + time_t lastref, + time_t lastmod, + uint32_t refcount, + uint16_t flags); + bool evictStaleAndContinue(const cache_key *candidateKey, const time_t maxRef, int &staleCount); int getNextFile(sfileno *, int *size); - StoreEntry *currentEntry() const; - void currentEntry(StoreEntry *); - StoreEntry *e; bool fromLog; bool _done; /// \bug (callback) should be hidden behind a proper human readable name diff -u -r -N squid-4.0.23/src/fs/ufs/UFSSwapDir.cc squid-4.0.24/src/fs/ufs/UFSSwapDir.cc --- squid-4.0.23/src/fs/ufs/UFSSwapDir.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/ufs/UFSSwapDir.cc 2018-03-08 02:16:46.000000000 +1300 @@ -509,7 +509,7 @@ ++removed; - e->release(); + e->release(true); } walker->Done(walker); @@ -801,9 +801,7 @@ e = new StoreEntry(); e->store_status = STORE_OK; e->setMemStatus(NOT_IN_MEMORY); - e->swap_status = SWAPOUT_DONE; - e->swap_filen = file_number; - e->swap_dirn = index; + e->attachToDisk(index, file_number, SWAPOUT_DONE); e->swap_file_sz = swap_file_sz; e->lastref = lastref; e->timestamp = timestamp; @@ -811,32 +809,17 @@ e->lastModified(lastmod); e->refcount = refcount; e->flags = newFlags; - EBIT_CLR(e->flags, RELEASE_REQUEST); - e->clearPrivate(); e->ping_status = PING_NONE; EBIT_CLR(e->flags, ENTRY_VALIDATED); mapBitSet(e->swap_filen); cur_size += fs.blksize * sizeInBlocks(e->swap_file_sz); ++n_disk_objects; - e->hashInsert(key); /* do it after we clear KEY_PRIVATE */ + e->hashInsert(key); replacementAdd (e); return e; } void -Fs::Ufs::UFSSwapDir::undoAddDiskRestore(StoreEntry *e) -{ - debugs(47, 5, HERE << *e); - replacementRemove(e); // checks swap_dirn so do it before we invalidate it - // Do not unlink the file as it might be used by a subsequent entry. - mapBitReset(e->swap_filen); - e->swap_filen = -1; - e->swap_dirn = -1; - cur_size -= fs.blksize * sizeInBlocks(e->swap_file_sz); - --n_disk_objects; -} - -void Fs::Ufs::UFSSwapDir::rebuild() { ++StoreController::store_dirs_rebuilding; @@ -1212,20 +1195,30 @@ } void -Fs::Ufs::UFSSwapDir::unlink(StoreEntry & e) +Fs::Ufs::UFSSwapDir::evictCached(StoreEntry & e) { - debugs(79, 3, HERE << "dirno " << index << ", fileno "<< - std::setfill('0') << std::hex << std::uppercase << std::setw(8) << e.swap_filen); - if (e.swap_status == SWAPOUT_DONE) { + debugs(79, 3, e); + if (e.locked()) // somebody else may still be using this file + return; // nothing to do: our get() always returns nil + + if (!e.hasDisk()) + return; // see evictIfFound() + + if (e.swappedOut()) { cur_size -= fs.blksize * sizeInBlocks(e.swap_file_sz); --n_disk_objects; } replacementRemove(&e); mapBitReset(e.swap_filen); UFSSwapDir::unlinkFile(e.swap_filen); - e.swap_filen = -1; - e.swap_dirn = -1; - e.swap_status = SWAPOUT_NONE; + e.detachFromDisk(); +} + +void +Fs::Ufs::UFSSwapDir::evictIfFound(const cache_key *) +{ + // UFS disk entries always have (attached) StoreEntries so if we got here, + // the entry is not cached on disk and there is nothing for us to do. } void @@ -1238,8 +1231,7 @@ void Fs::Ufs::UFSSwapDir::replacementRemove(StoreEntry * e) { - if (e->swap_dirn < 0) - return; + assert(e->hasDisk()); SwapDirPointer SD = INDEXSD(e->swap_dirn); @@ -1291,13 +1283,21 @@ } void -Fs::Ufs::UFSSwapDir::swappedOut(const StoreEntry &e) +Fs::Ufs::UFSSwapDir::finalizeSwapoutSuccess(const StoreEntry &e) { cur_size += fs.blksize * sizeInBlocks(e.swap_file_sz); ++n_disk_objects; } void +Fs::Ufs::UFSSwapDir::finalizeSwapoutFailure(StoreEntry &entry) +{ + debugs(47, 5, entry); + // rely on the expected subsequent StoreEntry::release(), evictCached(), or + // a similar call to call unlink(), detachFromDisk(), etc. for the entry. +} + +void Fs::Ufs::UFSSwapDir::logEntry(const StoreEntry & e, int op) const { if (swaplog_fd < 0) { diff -u -r -N squid-4.0.23/src/fs/ufs/UFSSwapDir.h squid-4.0.24/src/fs/ufs/UFSSwapDir.h --- squid-4.0.23/src/fs/ufs/UFSSwapDir.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/fs/ufs/UFSSwapDir.h 2018-03-08 02:16:46.000000000 +1300 @@ -49,10 +49,10 @@ virtual void dump(StoreEntry &) const override; virtual bool doubleCheck(StoreEntry &) override; virtual bool unlinkdUseful() const override; - virtual void unlink(StoreEntry &) override; virtual void statfs(StoreEntry &) const override; virtual void maintain() override; - virtual void markForUnlink(StoreEntry &) override {} + virtual void evictCached(StoreEntry &) override; + virtual void evictIfFound(const cache_key *) override; virtual bool canStore(const StoreEntry &e, int64_t diskSpaceNeeded, int &load) const override; virtual void reference(StoreEntry &) override; virtual bool dereference(StoreEntry &) override; @@ -67,11 +67,15 @@ virtual void reconfigure() override; virtual int callback() override; virtual void sync() override; - virtual void swappedOut(const StoreEntry &e) override; + virtual void finalizeSwapoutSuccess(const StoreEntry &) override; + virtual void finalizeSwapoutFailure(StoreEntry &) override; virtual uint64_t currentSize() const override { return cur_size; } virtual uint64_t currentCount() const override { return n_disk_objects; } virtual ConfigOption *getOptionTree() const override; virtual bool smpAware() const override { return false; } + /// as long as ufs relies on the global store_table to index entries, + /// it is wrong to ask individual ufs cache_dirs whether they have an entry + virtual bool hasReadableEntry(const StoreEntry &) const override { return false; } void unlinkFile(sfileno f); // move down when unlink is a virtual method @@ -99,8 +103,6 @@ uint32_t refcount, uint16_t flags, int clean); - /// Undo the effects of UFSSwapDir::addDiskRestore(). - void undoAddDiskRestore(StoreEntry *e); int validFileno(sfileno filn, int flag) const; int mapBitAllocate(); diff -u -r -N squid-4.0.23/src/FwdState.cc squid-4.0.24/src/FwdState.cc --- squid-4.0.23/src/FwdState.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/FwdState.cc 2018-03-08 02:16:46.000000000 +1300 @@ -1270,6 +1270,7 @@ else #endif conn->local = request->client_addr; + conn->local.port(0); // let OS pick the source port to prevent address clashes // some flags need setting on the socket to use this address conn->flags |= COMM_DOBIND; conn->flags |= COMM_TRANSPARENT; diff -u -r -N squid-4.0.23/src/gopher.cc squid-4.0.24/src/gopher.cc --- squid-4.0.23/src/gopher.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/gopher.cc 2018-03-08 02:16:46.000000000 +1300 @@ -922,7 +922,8 @@ CommIoCbPtrFun(gopherSendComplete, gopherState)); Comm::Write(gopherState->serverConn, &mb, call); - gopherState->entry->makePublic(); + if (!gopherState->entry->makePublic()) + gopherState->entry->makePrivate(true); } void diff -u -r -N squid-4.0.23/src/http/url_rewriters/LFS/url_lfs_rewrite.8 squid-4.0.24/src/http/url_rewriters/LFS/url_lfs_rewrite.8 --- squid-4.0.23/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2018-01-20 02:38:24.000000000 +1300 +++ squid-4.0.24/src/http/url_rewriters/LFS/url_lfs_rewrite.8 2018-03-08 02:33:26.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "URL_LFS_REWRITE 8" -.TH URL_LFS_REWRITE 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH URL_LFS_REWRITE 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/http.cc squid-4.0.24/src/http.cc --- squid-4.0.23/src/http.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/http.cc 2018-03-08 02:16:46.000000000 +1300 @@ -108,7 +108,7 @@ * for example, the request to this neighbor fails. */ if (_peer->options.proxy_only) - entry->releaseRequest(); + entry->releaseRequest(true); #if USE_DELAY_POOLS entry->setNoDelay(_peer->options.no_delay); @@ -252,7 +252,7 @@ #if USE_HTCP neighborsHtcpClear(e, NULL, e->mem_obj->request, e->mem_obj->method, HTCP_CLR_INVALIDATION); #endif - pe->release(); + pe->release(true); } /** \par @@ -269,7 +269,7 @@ #if USE_HTCP neighborsHtcpClear(e, NULL, e->mem_obj->request, HttpRequestMethod(Http::METHOD_HEAD), HTCP_CLR_INVALIDATION); #endif - pe->release(); + pe->release(true); } } @@ -334,7 +334,7 @@ #endif if (EBIT_TEST(entry->flags, RELEASE_REQUEST)) - return decision.make(ReuseDecision::reuseNot, "the entry has been released"); + return decision.make(ReuseDecision::doNotCacheButShare, "the entry has been released"); // RFC 7234 section 4: a cache MUST use the most recent response // (as determined by the Date header field) @@ -905,8 +905,11 @@ /* Check if object is cacheable or not based on reply code */ debugs(11, 3, "HTTP CODE: " << statusCode); - if (const StoreEntry *oldEntry = findPreviouslyCachedEntry(entry)) + if (StoreEntry *oldEntry = findPreviouslyCachedEntry(entry)) { + oldEntry->lock("HttpStateData::haveParsedReplyHeaders"); sawDateGoBack = rep->olderThan(oldEntry->getReply()); + oldEntry->unlock("HttpStateData::haveParsedReplyHeaders"); + } if (neighbors_do_private_keys && !sawDateGoBack) httpMaybeRemovePublic(entry, rep->sline.status()); @@ -954,11 +957,17 @@ break; case ReuseDecision::cachePositively: - entry->makePublic(); + if (!entry->makePublic()) { + decision.make(ReuseDecision::doNotCacheButShare, "public key creation error"); + entry->makePrivate(true); + } break; case ReuseDecision::cacheNegatively: - entry->cacheNegatively(); + if (!entry->cacheNegatively()) { + decision.make(ReuseDecision::doNotCacheButShare, "public key creation error"); + entry->makePrivate(true); + } break; case ReuseDecision::doNotCacheButShare: diff -u -r -N squid-4.0.23/src/ip/Address.cc squid-4.0.24/src/ip/Address.cc --- squid-4.0.23/src/ip/Address.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ip/Address.cc 2018-03-08 02:16:46.000000000 +1300 @@ -349,13 +349,6 @@ return false; } -Ip::Address& -Ip::Address::operator =(const Ip::Address &s) -{ - memcpy(this, &s, sizeof(Ip::Address)); - return *this; -}; - Ip::Address::Address(const char*s) { setEmpty(); @@ -440,7 +433,7 @@ { /* some AF_* magic to tell socket types apart and what we need to do */ if (s.ss_family == AF_INET6) { - memcpy(&mSocketAddr_, &s, sizeof(struct sockaddr_in6)); + memmove(&mSocketAddr_, &s, sizeof(struct sockaddr_in6)); } else { // convert it to our storage mapping. struct sockaddr_in *sin = (struct sockaddr_in*)&s; mSocketAddr_.sin6_port = sin->sin_port; @@ -458,8 +451,7 @@ Ip::Address & Ip::Address::operator =(struct sockaddr_in6 const &s) { - memcpy(&mSocketAddr_, &s, sizeof(struct sockaddr_in6)); - + memmove(&mSocketAddr_, &s, sizeof(struct sockaddr_in6)); return *this; }; @@ -486,19 +478,12 @@ Ip::Address & Ip::Address::operator =(struct in6_addr const &s) { - - memcpy(&mSocketAddr_.sin6_addr, &s, sizeof(struct in6_addr)); + memmove(&mSocketAddr_.sin6_addr, &s, sizeof(struct in6_addr)); mSocketAddr_.sin6_family = AF_INET6; return *this; }; -Ip::Address::Address(const Ip::Address &s) -{ - setEmpty(); - operator=(s); -} - Ip::Address::Address(const struct hostent &s) { setEmpty(); @@ -957,7 +942,7 @@ void Ip::Address::getSockAddr(struct sockaddr_in6 &buf) const { - memcpy(&buf, &mSocketAddr_, sizeof(struct sockaddr_in6)); + memmove(&buf, &mSocketAddr_, sizeof(struct sockaddr_in6)); /* maintain address family. It may have changed inside us. */ buf.sin6_family = AF_INET6; @@ -1005,7 +990,7 @@ void Ip::Address::getInAddr(struct in6_addr &buf) const { - memcpy(&buf, &mSocketAddr_.sin6_addr, sizeof(struct in6_addr)); + memmove(&buf, &mSocketAddr_.sin6_addr, sizeof(struct in6_addr)); } bool diff -u -r -N squid-4.0.23/src/ip/Address.h squid-4.0.24/src/ip/Address.h --- squid-4.0.23/src/ip/Address.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ip/Address.h 2018-03-08 02:16:46.000000000 +1300 @@ -44,7 +44,6 @@ /** @name Constructors and Destructor */ /*@{*/ Address() { setEmpty(); } - Address(const Ip::Address &); Address(const struct in_addr &); Address(const struct sockaddr_in &); Address(const struct in6_addr &); @@ -57,7 +56,6 @@ /** @name Assignment Operators */ /*@{*/ - Address& operator =(const Address &s); Address& operator =(struct sockaddr_in const &s); Address& operator =(struct sockaddr_storage const &s); Address& operator =(struct in_addr const &s); diff -u -r -N squid-4.0.23/src/ip/Intercept.cc squid-4.0.24/src/ip/Intercept.cc --- squid-4.0.23/src/ip/Intercept.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ip/Intercept.cc 2018-03-08 02:16:46.000000000 +1300 @@ -169,7 +169,6 @@ /* Trust the user configured properly. If not no harm done. * We will simply attempt a bind outgoing on our own IP. */ - newConn->remote.port(0); // allow random outgoing port to prevent address clashes debugs(89, 5, HERE << "address TPROXY: " << newConn); return true; #else diff -u -r -N squid-4.0.23/src/ipc/Kid.cc squid-4.0.24/src/ipc/Kid.cc --- squid-4.0.23/src/ipc/Kid.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ipc/Kid.cc 2018-03-08 02:16:46.000000000 +1300 @@ -90,7 +90,9 @@ if (hopeless() && Config.hopelessKidRevivalDelay) { syslog(LOG_NOTICE, "Squid Parent: %s process %d will not be restarted for %ld " "seconds due to repeated, frequent failures", - theName.termedBuf(), pid, Config.hopelessKidRevivalDelay); + theName.termedBuf(), + pid, + static_cast(Config.hopelessKidRevivalDelay)); } } diff -u -r -N squid-4.0.23/src/ipc/StoreMap.cc squid-4.0.24/src/ipc/StoreMap.cc --- squid-4.0.23/src/ipc/StoreMap.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ipc/StoreMap.cc 2018-03-08 02:16:46.000000000 +1300 @@ -12,6 +12,7 @@ #include "ipc/StoreMap.h" #include "sbuf/SBuf.h" #include "Store.h" +#include "store/Controller.h" #include "store_key_md5.h" #include "tools.h" @@ -212,6 +213,7 @@ debugs(54, 5, "closed clean entry " << fileno << " for writing " << path); } else { s.waitingToBeFreed = true; + s.writerHalted = true; s.lock.unlockExclusive(); debugs(54, 5, "closed dirty entry " << fileno << " for writing " << path); } @@ -253,17 +255,22 @@ return anchorAt(fileno); } -void +bool Ipc::StoreMap::freeEntry(const sfileno fileno) { debugs(54, 5, "marking entry " << fileno << " to be freed in " << path); Anchor &s = anchorAt(fileno); - if (s.lock.lockExclusive()) + if (s.lock.lockExclusive()) { + const bool result = !s.waitingToBeFreed && !s.empty(); freeChain(fileno, s, false); - else - s.waitingToBeFreed = true; // mark to free it later + return result; + } + + uint8_t expected = false; + // mark to free the locked entry later (if not already marked) + return s.waitingToBeFreed.compare_exchange_strong(expected, true); } void @@ -290,6 +297,25 @@ } } +bool +Ipc::StoreMap::markedForDeletion(const cache_key *const key) +{ + const int idx = fileNoByKey(key); + const Anchor &s = anchorAt(idx); + return s.sameKey(key) ? bool(s.waitingToBeFreed) : false; +} + +bool +Ipc::StoreMap::hasReadableEntry(const cache_key *const key) +{ + sfileno index; + if (openForReading(reinterpret_cast(key), index)) { + closeForReading(index); + return true; + } + return false; +} + /// unconditionally frees an already locked chain of slots, unlocking if needed void Ipc::StoreMap::freeChain(const sfileno fileno, Anchor &inode, const bool keepLocked) @@ -671,6 +697,7 @@ sfileno Ipc::StoreMap::nameByKey(const cache_key *const key) const { + assert(key); const uint64_t *const k = reinterpret_cast(key); // TODO: use a better hash function const int hash = (k[0] + k[1]) % entryLimit(); @@ -734,6 +761,7 @@ Ipc::StoreMapAnchor::setKey(const cache_key *const aKey) { memcpy(key, aKey, sizeof(key)); + waitingToBeFreed = Store::Root().markedForDeletion(aKey); } bool @@ -744,17 +772,36 @@ } void -Ipc::StoreMapAnchor::set(const StoreEntry &from) +Ipc::StoreMapAnchor::set(const StoreEntry &from, const cache_key *aKey) { assert(writing() && !reading()); - memcpy(key, from.key, sizeof(key)); + setKey(reinterpret_cast(aKey ? aKey : from.key)); basics.timestamp = from.timestamp; basics.lastref = from.lastref; basics.expires = from.expires; basics.lastmod = from.lastModified(); basics.swap_file_sz = from.swap_file_sz; basics.refcount = from.refcount; - basics.flags = from.flags; + + // do not copy key bit if we are not using from.key + // TODO: Replace KEY_PRIVATE with a nil StoreEntry::key! + uint16_t cleanFlags = from.flags; + if (aKey) + EBIT_CLR(cleanFlags, KEY_PRIVATE); + basics.flags = cleanFlags; +} + +void +Ipc::StoreMapAnchor::exportInto(StoreEntry &into) const +{ + assert(reading()); + into.timestamp = basics.timestamp; + into.lastref = basics.lastref; + into.expires = basics.expires; + into.lastModified(basics.lastmod); + into.swap_file_sz = basics.swap_file_sz; + into.refcount = basics.refcount; + into.flags = basics.flags; } void @@ -766,6 +813,7 @@ memset(&key, 0, sizeof(key)); memset(&basics, 0, sizeof(basics)); waitingToBeFreed = false; + writerHalted = false; // but keep the lock } diff -u -r -N squid-4.0.23/src/ipc/StoreMap.h squid-4.0.24/src/ipc/StoreMap.h --- squid-4.0.23/src/ipc/StoreMap.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ipc/StoreMap.h 2018-03-08 02:16:46.000000000 +1300 @@ -56,7 +56,9 @@ StoreMapAnchor(); /// store StoreEntry key and basics for an inode slot - void set(const StoreEntry &anEntry); + void set(const StoreEntry &anEntry, const cache_key *aKey = nullptr); + /// load StoreEntry basics that were previously stored with set() + void exportInto(StoreEntry &) const; void setKey(const cache_key *const aKey); bool sameKey(const cache_key *const aKey) const; @@ -74,6 +76,8 @@ public: mutable ReadWriteLock lock; ///< protects slot data below std::atomic waitingToBeFreed; ///< may be accessed w/o a lock + /// whether StoreMap::abortWriting() was called for a read-locked entry + std::atomic writerHalted; // fields marked with [app] can be modified when appending-while-reading // fields marked with [update] can be modified when updating-while-reading @@ -248,11 +252,19 @@ const Anchor &peekAtEntry(const sfileno fileno) const; /// free the entry if possible or mark it as waiting to be freed if not - void freeEntry(const sfileno fileno); + /// \returns whether the entry was neither empty nor marked + bool freeEntry(const sfileno); /// free the entry if possible or mark it as waiting to be freed if not /// does nothing if we cannot check that the key matches the cached entry void freeEntryByKey(const cache_key *const key); + /// whether the entry with the given key exists and was marked as + /// "waiting to be freed" some time ago + bool markedForDeletion(const cache_key *const); + + /// whether the index contains a valid readable entry with the given key + bool hasReadableEntry(const cache_key *const); + /// opens entry (identified by key) for reading, increments read level const Anchor *openForReading(const cache_key *const key, sfileno &fileno); /// opens entry (identified by sfileno) for reading, increments read level diff -u -r -N squid-4.0.23/src/log/DB/log_db_daemon.8 squid-4.0.24/src/log/DB/log_db_daemon.8 --- squid-4.0.23/src/log/DB/log_db_daemon.8 2018-01-20 02:38:24.000000000 +1300 +++ squid-4.0.24/src/log/DB/log_db_daemon.8 2018-03-08 02:33:26.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "LOG_DB_DAEMON 8" -.TH LOG_DB_DAEMON 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH LOG_DB_DAEMON 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/MemBuf.cc squid-4.0.24/src/MemBuf.cc --- squid-4.0.23/src/MemBuf.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/MemBuf.cc 2018-03-08 02:16:46.000000000 +1300 @@ -76,15 +76,6 @@ #include "MemBuf.h" #include "profiler/Profiler.h" -#ifdef VA_COPY -#undef VA_COPY -#endif -#if defined HAVE_VA_COPY -#define VA_COPY va_copy -#elif defined HAVE___VA_COPY -#define VA_COPY __va_copy -#endif - /* local constants */ /* default values for buffer sizes, used by memBufDefInit */ @@ -268,10 +259,6 @@ void MemBuf::vappendf(const char *fmt, va_list vargs) { -#ifdef VA_COPY - va_list ap; -#endif - int sz = 0; assert(fmt); assert(buf); @@ -282,18 +269,15 @@ mb_size_t free_space = capacity - size; /* put as much as we can */ -#ifdef VA_COPY /* Fix of bug 753r. The value of vargs is undefined * after vsnprintf() returns. Make a copy of vargs * incase we loop around and call vsnprintf() again. */ - VA_COPY(ap,vargs); + va_list ap; + va_copy(ap,vargs); sz = vsnprintf(buf + size, free_space, fmt, ap); va_end(ap); -#else /* VA_COPY */ - sz = vsnprintf(buf + size, free_space, fmt, vargs); -#endif /*VA_COPY*/ /* check for possible overflow */ /* snprintf on Linuz returns -1 on overflows */ /* snprintf on FreeBSD returns at least free_space on overflows */ diff -u -r -N squid-4.0.23/src/MemObject.cc squid-4.0.24/src/MemObject.cc --- squid-4.0.23/src/MemObject.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/MemObject.cc 2018-03-08 02:16:46.000000000 +1300 @@ -99,7 +99,6 @@ MemObject::MemObject() : inmem_lo(0), nclients(0), - smpCollapsed(false), request(nullptr), ping_reply_callback(nullptr), ircb_data(nullptr), @@ -247,8 +246,6 @@ mb->appendf("\tmem-cache index: %d state: %d offset: %" PRId64 "\n", memCache.index, memCache.io, memCache.offset); if (object_sz >= 0) mb->appendf("\tobject_sz: %" PRId64 "\n", object_sz); - if (smpCollapsed) - mb->appendf("\tsmp-collapsed\n"); StoreClientStats statsVisitor(mb); diff -u -r -N squid-4.0.23/src/MemObject.h squid-4.0.24/src/MemObject.h --- squid-4.0.23/src/MemObject.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/MemObject.h 2018-03-08 02:16:46.000000000 +1300 @@ -16,6 +16,7 @@ #include "sbuf/SBuf.h" #include "SquidString.h" #include "stmem.h" +#include "store/forward.h" #include "StoreIOBuffer.h" #include "StoreIOState.h" #include "typedefs.h" //for IRCB @@ -126,8 +127,12 @@ SwapOut swapout; - /// cache "I/O" direction and status - typedef enum { ioUndecided, ioWriting, ioReading, ioDone } Io; + /* TODO: Remove this change-minimizing hack */ + using Io = Store::IoStatus; + static constexpr Io ioUndecided = Store::ioUndecided; + static constexpr Io ioReading = Store::ioReading; + static constexpr Io ioWriting = Store::ioWriting; + static constexpr Io ioDone = Store::ioDone; /// State of an entry with regards to the [shared] in-transit table. class XitTable @@ -153,8 +158,6 @@ }; MemCache memCache; ///< current [shared] memory caching state for the entry - bool smpCollapsed; ///< whether this entry gets data from another worker - /* Read only - this reply must be preserved by store clients */ /* The original reply. possibly with updated metadata. */ HttpRequest *request; diff -u -r -N squid-4.0.23/src/MemStore.cc squid-4.0.24/src/MemStore.cc --- squid-4.0.23/src/MemStore.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/MemStore.cc 2018-03-08 02:16:46.000000000 +1300 @@ -108,14 +108,10 @@ ShmWriter::vappendf(const char *fmt, va_list ap) { SBuf vaBuf; -#if defined(VA_COPY) va_list apCopy; - VA_COPY(apCopy, ap); + va_copy(apCopy, ap); vaBuf.vappendf(fmt, apCopy); va_end(apCopy); -#else - vaBuf.vappendf(fmt, ap); -#endif append(vaBuf.rawContent(), vaBuf.length()); } @@ -321,7 +317,7 @@ StoreEntry *e = new StoreEntry(); // XXX: We do not know the URLs yet, only the key, but we need to parse and - // store the response for the Root().get() callers to be happy because they + // store the response for the Root().find() callers to be happy because they // expect IN_MEMORY entries to already have the response headers and body. e->createMemObject(); @@ -329,13 +325,12 @@ const bool copied = copyFromShm(*e, index, *slot); - if (copied) { - e->hashInsert(key); + if (copied) return e; - } - debugs(20, 3, HERE << "mem-loading failed; freeing " << index); + debugs(20, 3, "failed for " << *e); map->freeEntry(index); // do not let others into the same trap + destroyStoreEntry(static_cast(e)); return NULL; } @@ -403,46 +398,41 @@ } bool -MemStore::anchorCollapsed(StoreEntry &collapsed, bool &inSync) +MemStore::anchorToCache(StoreEntry &entry, bool &inSync) { if (!map) return false; sfileno index; const Ipc::StoreMapAnchor *const slot = map->openForReading( - reinterpret_cast(collapsed.key), index); + reinterpret_cast(entry.key), index); if (!slot) return false; - anchorEntry(collapsed, index, *slot); - inSync = updateCollapsedWith(collapsed, index, *slot); + anchorEntry(entry, index, *slot); + inSync = updateAnchoredWith(entry, index, *slot); return true; // even if inSync is false } bool -MemStore::updateCollapsed(StoreEntry &collapsed) +MemStore::updateAnchored(StoreEntry &entry) { - assert(collapsed.mem_obj); - - const sfileno index = collapsed.mem_obj->memCache.index; - - // already disconnected from the cache, no need to update - if (index < 0) - return true; - if (!map) return false; + assert(entry.mem_obj); + assert(entry.hasMemStore()); + const sfileno index = entry.mem_obj->memCache.index; const Ipc::StoreMapAnchor &anchor = map->readableEntry(index); - return updateCollapsedWith(collapsed, index, anchor); + return updateAnchoredWith(entry, index, anchor); } -/// updates collapsed entry after its anchor has been located +/// updates Transients entry after its anchor has been located bool -MemStore::updateCollapsedWith(StoreEntry &collapsed, const sfileno index, const Ipc::StoreMapAnchor &anchor) +MemStore::updateAnchoredWith(StoreEntry &entry, const sfileno index, const Ipc::StoreMapAnchor &anchor) { - collapsed.swap_file_sz = anchor.basics.swap_file_sz; - const bool copied = copyFromShm(collapsed, index, anchor); + entry.swap_file_sz = anchor.basics.swap_file_sz; + const bool copied = copyFromShm(entry, index, anchor); return copied; } @@ -450,15 +440,8 @@ void MemStore::anchorEntry(StoreEntry &e, const sfileno index, const Ipc::StoreMapAnchor &anchor) { - const Ipc::StoreMapAnchor::Basics &basics = anchor.basics; - - e.swap_file_sz = basics.swap_file_sz; - e.lastref = basics.lastref; - e.timestamp = basics.timestamp; - e.expires = basics.expires; - e.lastModified(basics.lastmod); - e.refcount = basics.refcount; - e.flags = basics.flags; + assert(!e.hasDisk()); // no conflict with disk entry basics + anchor.exportInto(e); assert(e.mem_obj); if (anchor.complete()) { @@ -470,11 +453,7 @@ assert(e.mem_obj->object_sz < 0); e.setMemStatus(NOT_IN_MEMORY); } - assert(e.swap_status == SWAPOUT_NONE); // set in StoreEntry constructor - e.ping_status = PING_NONE; - EBIT_CLR(e.flags, RELEASE_REQUEST); - e.clearPrivate(); EBIT_SET(e.flags, ENTRY_VALIDATED); MemObject::MemCache &mc = e.mem_obj->memCache; @@ -910,25 +889,29 @@ } void -MemStore::markForUnlink(StoreEntry &e) +MemStore::evictCached(StoreEntry &e) { - assert(e.mem_obj); - if (e.mem_obj->memCache.index >= 0) - map->freeEntry(e.mem_obj->memCache.index); + debugs(47, 5, e); + if (e.hasMemStore()) { + if (map->freeEntry(e.mem_obj->memCache.index)) + CollapsedForwarding::Broadcast(e); + if (!e.locked()) { + disconnect(e); + e.destroyMemObject(); + } + } else if (const auto key = e.publicKey()) { + // the entry may have been loaded and then disconnected from the cache + evictIfFound(key); + if (!e.locked()) + e.destroyMemObject(); + } } void -MemStore::unlink(StoreEntry &e) +MemStore::evictIfFound(const cache_key *key) { - if (e.mem_obj && e.mem_obj->memCache.index >= 0) { - map->freeEntry(e.mem_obj->memCache.index); - disconnect(e); - } else if (map) { - // the entry may have been loaded and then disconnected from the cache - map->freeEntryByKey(reinterpret_cast(e.key)); - } - - e.destroyMemObject(); // XXX: but it may contain useful info such as a client list. The old code used to do that though, right? + if (map) + map->freeEntryByKey(key); } void @@ -936,12 +919,12 @@ { assert(e.mem_obj); MemObject &mem_obj = *e.mem_obj; - if (mem_obj.memCache.index >= 0) { + if (e.hasMemStore()) { if (mem_obj.memCache.io == MemObject::ioWriting) { map->abortWriting(mem_obj.memCache.index); mem_obj.memCache.index = -1; mem_obj.memCache.io = MemObject::ioDone; - Store::Root().transientsAbandon(e); // broadcasts after the change + Store::Root().stopSharing(e); // broadcasts after the change } else { assert(mem_obj.memCache.io == MemObject::ioReading); map->closeForReading(mem_obj.memCache.index); diff -u -r -N squid-4.0.23/src/MemStore.h squid-4.0.24/src/MemStore.h --- squid-4.0.23/src/MemStore.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/MemStore.h 2018-03-08 02:16:46.000000000 +1300 @@ -59,10 +59,10 @@ virtual bool dereference(StoreEntry &e) override; virtual void updateHeaders(StoreEntry *e) override; virtual void maintain() override; - virtual bool anchorCollapsed(StoreEntry &e, bool &inSync) override; - virtual bool updateCollapsed(StoreEntry &e) override; - virtual void markForUnlink(StoreEntry &) override; - virtual void unlink(StoreEntry &e) override; + virtual bool anchorToCache(StoreEntry &e, bool &inSync) override; + virtual bool updateAnchored(StoreEntry &) override; + virtual void evictCached(StoreEntry &) override; + virtual void evictIfFound(const cache_key *) override; virtual bool smpAware() const override { return true; } static int64_t EntryLimit(); @@ -81,7 +81,7 @@ void updateHeadersOrThrow(Ipc::StoreMapUpdate &update); void anchorEntry(StoreEntry &e, const sfileno index, const Ipc::StoreMapAnchor &anchor); - bool updateCollapsedWith(StoreEntry &collapsed, const sfileno index, const Ipc::StoreMapAnchor &anchor); + bool updateAnchoredWith(StoreEntry &, const sfileno, const Ipc::StoreMapAnchor &); Ipc::Mem::PageId pageForSlice(Ipc::StoreMapSliceId sliceId); Ipc::StoreMap::Slice &nextAppendableSlice(const sfileno entryIndex, sfileno &sliceOffset); diff -u -r -N squid-4.0.23/src/mime.cc squid-4.0.24/src/mime.cc --- squid-4.0.23/src/mime.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/mime.cc 2018-03-08 02:16:46.000000000 +1300 @@ -19,7 +19,6 @@ #include "MemBuf.h" #include "MemObject.h" #include "mime.h" -#include "RequestFlags.h" #include "SquidConfig.h" #include "Store.h" #include "StoreClient.h" @@ -394,18 +393,19 @@ status = Http::scNoContent; } + StoreEntry *e = storeCreatePureEntry(url_, url_, Http::METHOD_GET); + e->lock("MimeIcon::created"); + EBIT_SET(e->flags, ENTRY_SPECIAL); + const auto madePublic = e->setPublicKey(); + assert(madePublic); // nothing can block ENTRY_SPECIAL from becoming public + + /* fill `e` with a canned 2xx response object */ + const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initIcon); HttpRequest *r = HttpRequest::FromUrl(url_, mx); if (!r) fatalf("mimeLoadIcon: cannot parse internal URL: %s", url_); - // fill newEntry with a canned 2xx response object - RequestFlags flags; - flags.cachable = true; - StoreEntry *e = storeCreateEntry(url_,url_,flags,Http::METHOD_GET); - assert(e != NULL); - EBIT_SET(e->flags, ENTRY_SPECIAL); - e->setPublicKey(); e->buffer(); e->mem_obj->request = r; diff -u -r -N squid-4.0.23/src/neighbors.cc squid-4.0.24/src/neighbors.cc --- squid-4.0.23/src/neighbors.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/neighbors.cc 2018-03-08 02:16:46.000000000 +1300 @@ -593,7 +593,7 @@ if (Config.peers == NULL) return 0; - assert(entry->swap_status == SWAPOUT_NONE); + assert(!entry->hasDisk()); mem->start_ping = current_time; @@ -984,7 +984,7 @@ debugs(15, 6, "neighborsUdpAck: opcode " << opcode << " '" << storeKeyText(key) << "'"); - if (NULL != (entry = Store::Root().get(key))) + if ((entry = Store::Root().findCallback(key))) mem = entry->mem_obj; if ((p = whichPeer(from))) @@ -1692,7 +1692,7 @@ void neighborsHtcpReply(const cache_key * key, HtcpReplyData * htcp, const Ip::Address &from) { - StoreEntry *e = Store::Root().get(key); + StoreEntry *e = Store::Root().findCallback(key); MemObject *mem = NULL; CachePeer *p; peer_t ntype = PEER_NONE; diff -u -r -N squid-4.0.23/src/peer_digest.cc squid-4.0.24/src/peer_digest.cc --- squid-4.0.23/src/peer_digest.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/peer_digest.cc 2018-03-08 02:16:46.000000000 +1300 @@ -305,7 +305,6 @@ CachePeer *p = pd->peer; StoreEntry *e, *old_e; char *url = NULL; - const cache_key *key; HttpRequest *req; StoreIOBuffer tempBuffer; @@ -318,16 +317,13 @@ url = xstrdup(p->digest_url); else url = xstrdup(internalRemoteUri(p->host, p->http_port, "/squid-internal-periodic/", SBuf(StoreDigestFileName))); + debugs(72, 2, url); const MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initCacheDigest); req = HttpRequest::FromUrl(url, mx); assert(req); - key = storeKeyPublicByRequest(req); - - debugs(72, 2, "peerDigestRequest: " << url << " key: " << storeKeyText(key)); - /* add custom headers */ assert(!req->header.len); @@ -351,21 +347,22 @@ pd_last_req_time = squid_curtime; req->flags.cachable = true; - /* the rest is based on clientProcessExpired() */ + /* the rest is based on clientReplyContext::processExpired() */ req->flags.refresh = true; - old_e = fetch->old_entry = Store::Root().get(key); + old_e = fetch->old_entry = storeGetPublicByRequest(req); if (old_e) { - debugs(72, 5, "peerDigestRequest: found old entry"); + debugs(72, 5, "found old " << *old_e); old_e->lock("peerDigestRequest"); - old_e->createMemObject(url, url, req->method); + old_e->ensureMemObject(url, url, req->method); fetch->old_sc = storeClientListAdd(old_e, fetch); } e = fetch->entry = storeCreateEntry(url, url, req->flags, req->method); + debugs(72, 5, "created " << *e); assert(EBIT_TEST(e->flags, KEY_PRIVATE)); fetch->sc = storeClientListAdd(e, fetch); /* set lastmod to trigger IMS request if possible */ @@ -374,8 +371,6 @@ e->lastModified(old_e->lastModified()); /* push towards peer cache */ - debugs(72, 3, "peerDigestRequest: forwarding to fwdStart..."); - FwdState::fwdStart(Comm::ConnectionPointer(), e, req); tempBuffer.offset = 0; diff -u -r -N squid-4.0.23/src/peer_select.cc squid-4.0.24/src/peer_select.cc --- squid-4.0.23/src/peer_select.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/peer_select.cc 2018-03-08 02:16:46.000000000 +1300 @@ -953,7 +953,7 @@ // Non-PINNED destinations are uniquely identified by their CachePeer // (even though a DIRECT destination might match a cache_peer address). const bool duplicate = (server->code == PINNED) ? - (code == PINNED) : (server->_peer == peer); + (code == PINNED) : (server->_peer == peer); if (duplicate) { debugs(44, 3, "skipping " << PeerSelectionDumper(ps, peer, code) << "; have " << PeerSelectionDumper(ps, server->_peer.get(), server->code)); diff -u -r -N squid-4.0.23/src/sbuf/SBuf.cc squid-4.0.24/src/sbuf/SBuf.cc --- squid-4.0.23/src/sbuf/SBuf.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/sbuf/SBuf.cc 2018-03-08 02:16:46.000000000 +1300 @@ -19,15 +19,6 @@ #include #include -#ifdef VA_COPY -#undef VA_COPY -#endif -#if defined HAVE_VA_COPY -#define VA_COPY va_copy -#elif defined HAVE___VA_COPY -#define VA_COPY __va_copy -#endif - InstanceIdDefinitions(SBuf, "SBuf"); SBufStats SBuf::stats; @@ -266,14 +257,10 @@ size_type requiredSpaceEstimate = strlen(fmt)*2; char *space = rawSpace(requiredSpaceEstimate); -#ifdef VA_COPY va_list ap; - VA_COPY(ap, vargs); + va_copy(ap, vargs); sz = vsnprintf(space, spaceSize(), fmt, ap); va_end(ap); -#else - sz = vsnprintf(space, spaceSize(), fmt, vargs); -#endif Must2(sz >= 0, "vsnprintf() output error"); /* check for possible overflow */ diff -u -r -N squid-4.0.23/src/sbuf/SBuf.h squid-4.0.24/src/sbuf/SBuf.h --- squid-4.0.23/src/sbuf/SBuf.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/sbuf/SBuf.h 2018-03-08 02:16:46.000000000 +1300 @@ -20,7 +20,6 @@ #include "sbuf/Stats.h" #include -#include #include #include #if HAVE_UNISTD_H diff -u -r -N squid-4.0.23/src/security/cert_generators/file/security_file_certgen.cc squid-4.0.24/src/security/cert_generators/file/security_file_certgen.cc --- squid-4.0.23/src/security/cert_generators/file/security_file_certgen.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/cert_generators/file/security_file_certgen.cc 2018-03-08 02:16:46.000000000 +1300 @@ -313,11 +313,11 @@ struct statvfs sfs; if (xstatvfs(db_path.c_str(), &sfs)) { - fs_block_size = 2048; + fs_block_size = 2048; } else { - fs_block_size = sfs.f_frsize; - // Sanity check; make sure we have a meaningful value. - if (fs_block_size < 512) + fs_block_size = sfs.f_frsize; + // Sanity check; make sure we have a meaningful value. + if (fs_block_size < 512) fs_block_size = 2048; } } diff -u -r -N squid-4.0.23/src/security/cert_validators/fake/security_fake_certverify.8 squid-4.0.24/src/security/cert_validators/fake/security_fake_certverify.8 --- squid-4.0.23/src/security/cert_validators/fake/security_fake_certverify.8 2018-01-20 02:38:24.000000000 +1300 +++ squid-4.0.24/src/security/cert_validators/fake/security_fake_certverify.8 2018-03-08 02:33:27.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "SECURITY_FAKE_CERTVERIFY 8" -.TH SECURITY_FAKE_CERTVERIFY 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH SECURITY_FAKE_CERTVERIFY 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/security/forward.h squid-4.0.24/src/security/forward.h --- squid-4.0.23/src/security/forward.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/forward.h 2018-03-08 02:16:46.000000000 +1300 @@ -13,8 +13,8 @@ #include "security/Context.h" #include "security/Session.h" -#if USE_GNUTLS && HAVE_GNUTLS_X509_H -#include +#if USE_GNUTLS && HAVE_GNUTLS_ABSTRACT_H +#include #endif #include #if USE_OPENSSL && HAVE_OPENSSL_ERR_H @@ -86,10 +86,9 @@ CtoCpp1(X509_free, X509 *) typedef Security::LockingPointer > CertPointer; #elif USE_GNUTLS -CtoCpp1(gnutls_x509_crt_deinit, gnutls_x509_crt_t) -typedef Security::LockingPointer CertPointer; +typedef std::shared_ptr CertPointer; #else -typedef void * CertPointer; +typedef std::shared_ptr CertPointer; #endif #if USE_OPENSSL @@ -167,9 +166,10 @@ #if USE_OPENSSL CtoCpp1(EVP_PKEY_free, EVP_PKEY *) typedef Security::LockingPointer > PrivateKeyPointer; +#elif USE_GNUTLS +typedef std::shared_ptr PrivateKeyPointer; #else -// XXX: incompatible with the other PrivateKeyPointer declaration (lacks self-initialization) -typedef void *PrivateKeyPointer; +typedef std::shared_ptr PrivateKeyPointer; #endif class ServerOptions; diff -u -r -N squid-4.0.23/src/security/KeyData.cc squid-4.0.24/src/security/KeyData.cc --- squid-4.0.23/src/security/KeyData.cc 1970-01-01 12:00:00.000000000 +1200 +++ squid-4.0.24/src/security/KeyData.cc 2018-03-08 02:16:46.000000000 +1300 @@ -0,0 +1,184 @@ +/* + * Copyright (C) 1996-2018 The Squid Software Foundation and contributors + * + * Squid software is distributed under GPLv2+ license and includes + * contributions from numerous individuals and organizations. + * Please see the COPYING and CONTRIBUTORS files for details. + */ + +#include "squid.h" +#include "anyp/PortCfg.h" +#include "fatal.h" +#include "security/KeyData.h" +#include "SquidConfig.h" +#include "ssl/bio.h" + +/** + * Read certificate from file. + * See also: Ssl::ReadX509Certificate function, gadgets.cc file + */ +bool +Security::KeyData::loadX509CertFromFile() +{ + debugs(83, DBG_IMPORTANT, "Using certificate in " << certFile); + cert.reset(); // paranoid: ensure cert is unset + +#if USE_OPENSSL + const char *certFilename = certFile.c_str(); + Ssl::BIO_Pointer bio(BIO_new(BIO_s_file())); + if (!bio || !BIO_read_filename(bio.get(), certFilename)) { + const auto x = ERR_get_error(); + debugs(83, DBG_IMPORTANT, "ERROR: unable to load certificate file '" << certFile << "': " << ErrorString(x)); + return false; + } + + if (X509 *certificate = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) { + cert.resetWithoutLocking(certificate); + } + +#elif USE_GNUTLS + const char *certFilename = certFile.c_str(); + gnutls_datum_t data; + Security::ErrorCode x = gnutls_load_file(certFilename, &data); + if (x != GNUTLS_E_SUCCESS) { + debugs(83, DBG_IMPORTANT, "ERROR: unable to load certificate file '" << certFile << "': " << ErrorString(x)); + return false; + } + + gnutls_pcert_st pcrt; + x = gnutls_pcert_import_x509_raw(&pcrt, &data, GNUTLS_X509_FMT_PEM, 0); + if (x != GNUTLS_E_SUCCESS) { + debugs(83, DBG_IMPORTANT, "ERROR: unable to import certificate from '" << certFile << "': " << ErrorString(x)); + return false; + } + gnutls_free(data.data); + + gnutls_x509_crt_t certificate; + x = gnutls_pcert_export_x509(&pcrt, &certificate); + if (x != GNUTLS_E_SUCCESS) { + debugs(83, DBG_IMPORTANT, "ERROR: unable to X.509 convert certificate from '" << certFile << "': " << ErrorString(x)); + return false; + } + + if (certificate) { + cert = Security::CertPointer(certificate, [](gnutls_x509_crt_t p) { + debugs(83, 5, "gnutls_x509_crt_deinit cert=" << (void*)p); + gnutls_x509_crt_deinit(p); + }); + } + +#else + // do nothing. +#endif + + if (!cert) { + debugs(83, DBG_IMPORTANT, "ERROR: unable to load certificate from '" << certFile << "'"); + } + + return bool(cert); +} + +/** + * Read certificate from file. + * See also: Ssl::ReadX509Certificate function, gadgets.cc file + */ +void +Security::KeyData::loadX509ChainFromFile() +{ +#if USE_OPENSSL + // XXX: This BIO loads the public cert as first chain cert, + // so the code appending chains sends it twice in handshakes. + const char *certFilename = certFile.c_str(); + Ssl::BIO_Pointer bio(BIO_new(BIO_s_file())); + if (!bio || !BIO_read_filename(bio.get(), certFilename)) { + const auto x = ERR_get_error(); + debugs(83, DBG_IMPORTANT, "ERROR: unable to load chain file '" << certFile << "': " << ErrorString(x)); + return; + } + + if (X509_check_issued(cert.get(), cert.get()) == X509_V_OK) + debugs(83, 5, "Certificate is self-signed, will not be chained"); + else { + // and add to the chain any other certificate exist in the file + while (X509 *ca = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) { + // XXX: self-signed check should be applied to all certs loaded. + // XXX: missing checks that the chained certs are actually part of a chain for validating cert. + chain.emplace_front(Security::CertPointer(ca)); + } + } + +#elif USE_GNUTLS + // XXX: implement chain loading + debugs(83, 2, "Loading certificate chain from PEM files not implemented in this Squid."); + +#else + // nothing to do. +#endif +} + +/** + * Read X.509 private key from file. + */ +bool +Security::KeyData::loadX509PrivateKeyFromFile() +{ + debugs(83, DBG_IMPORTANT, "Using key in " << privateKeyFile); + +#if USE_OPENSSL + const char *keyFilename = privateKeyFile.c_str(); + // XXX: Ssl::AskPasswordCb needs SSL_CTX_set_default_passwd_cb_userdata() + // so this may not fully work iff Config.Program.ssl_password is set. + pem_password_cb *cb = ::Config.Program.ssl_password ? &Ssl::AskPasswordCb : nullptr; + Ssl::ReadPrivateKeyFromFile(keyFilename, pkey, cb); + + if (pkey && !X509_check_private_key(cert.get(), pkey.get())) { + debugs(83, DBG_IMPORTANT, "WARNING: '" << privateKeyFile << "' X509_check_private_key() failed"); + pkey.reset(); + } + +#elif USE_GNUTLS + const char *keyFilename = privateKeyFile.c_str(); + gnutls_datum_t data; + if (gnutls_load_file(keyFilename, &data) == GNUTLS_E_SUCCESS) { + gnutls_privkey_t key; + (void)gnutls_privkey_init(&key); + Security::ErrorCode x = gnutls_privkey_import_x509_raw(key, &data, GNUTLS_X509_FMT_PEM, nullptr, 0); + if (x == GNUTLS_E_SUCCESS) { + gnutls_x509_privkey_t xkey; + gnutls_privkey_export_x509(key, &xkey); + gnutls_privkey_deinit(key); + pkey = Security::PrivateKeyPointer(xkey, [](gnutls_x509_privkey_t p) { + debugs(83, 5, "gnutls_x509_privkey_deinit pkey=" << (void*)p); + gnutls_x509_privkey_deinit(p); + }); + } + } + gnutls_free(data.data); + +#else + // nothing to do. +#endif + + return bool(pkey); +} + +void +Security::KeyData::loadFromFiles(const AnyP::PortCfg &port, const char *portType) +{ + char buf[128]; + if (!loadX509CertFromFile()) { + debugs(83, DBG_IMPORTANT, "WARNING: '" << portType << "_port " << port.s.toUrl(buf, sizeof(buf)) << "' missing certificate in '" << certFile << "'"); + return; + } + + // certificate chain in the PEM file is optional + loadX509ChainFromFile(); + + // pkey is mandatory, not having it makes cert and chain pointless. + if (!loadX509PrivateKeyFromFile()) { + debugs(83, DBG_IMPORTANT, "WARNING: '" << portType << "_port " << port.s.toUrl(buf, sizeof(buf)) << "' missing private key in '" << privateKeyFile << "'"); + cert.reset(); + chain.clear(); + } +} + diff -u -r -N squid-4.0.23/src/security/KeyData.h squid-4.0.24/src/security/KeyData.h --- squid-4.0.23/src/security/KeyData.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/KeyData.h 2018-03-08 02:16:46.000000000 +1300 @@ -9,6 +9,7 @@ #ifndef SQUID_SRC_SECURITY_KEYDATA_H #define SQUID_SRC_SECURITY_KEYDATA_H +#include "anyp/forward.h" #include "sbuf/SBuf.h" #include "security/forward.h" @@ -19,8 +20,24 @@ class KeyData { public: + /// load the contents of certFile and privateKeyFile into memory cert, pkey and chain + void loadFromFiles(const AnyP::PortCfg &, const char *portType); + +public: SBuf certFile; ///< path of file containing PEM format X.509 certificate SBuf privateKeyFile; ///< path of file containing private key in PEM format + + /// public X.509 certificate from certFile + Security::CertPointer cert; + /// private key from privateKeyFile + Security::PrivateKeyPointer pkey; + /// any certificates which must be chained from cert + Security::CertList chain; + +private: + bool loadX509CertFromFile(); + void loadX509ChainFromFile(); + bool loadX509PrivateKeyFromFile(); }; } // namespace Security diff -u -r -N squid-4.0.23/src/security/Makefile.am squid-4.0.24/src/security/Makefile.am --- squid-4.0.23/src/security/Makefile.am 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/Makefile.am 2018-03-08 02:16:46.000000000 +1300 @@ -22,6 +22,7 @@ Handshake.cc \ Handshake.h \ forward.h \ + KeyData.cc \ KeyData.h \ LockingPointer.h \ NegotiationHistory.cc \ diff -u -r -N squid-4.0.23/src/security/Makefile.in squid-4.0.24/src/security/Makefile.in --- squid-4.0.23/src/security/Makefile.in 2018-01-20 02:30:54.000000000 +1300 +++ squid-4.0.24/src/security/Makefile.in 2018-03-08 02:24:44.000000000 +1300 @@ -164,7 +164,7 @@ LTLIBRARIES = $(noinst_LTLIBRARIES) libsecurity_la_LIBADD = am_libsecurity_la_OBJECTS = BlindPeerConnector.lo EncryptorAnswer.lo \ - Handshake.lo NegotiationHistory.lo PeerConnector.lo \ + Handshake.lo KeyData.lo NegotiationHistory.lo PeerConnector.lo \ PeerOptions.lo ServerOptions.lo Session.lo libsecurity_la_OBJECTS = $(am_libsecurity_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) @@ -766,6 +766,7 @@ Handshake.cc \ Handshake.h \ forward.h \ + KeyData.cc \ KeyData.h \ LockingPointer.h \ NegotiationHistory.cc \ @@ -846,6 +847,7 @@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/BlindPeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/EncryptorAnswer.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/Handshake.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/KeyData.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/NegotiationHistory.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerConnector.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/PeerOptions.Plo@am__quote@ diff -u -r -N squid-4.0.23/src/security/PeerOptions.h squid-4.0.24/src/security/PeerOptions.h --- squid-4.0.23/src/security/PeerOptions.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/PeerOptions.h 2018-03-08 02:16:46.000000000 +1300 @@ -66,6 +66,7 @@ void parseOptions(); ///< parsed value of sslOptions long parseFlags(); void loadCrlFile(); + void loadKeysFile(); public: SBuf sslOptions; ///< library-specific options string diff -u -r -N squid-4.0.23/src/security/ServerOptions.cc squid-4.0.24/src/security/ServerOptions.cc --- squid-4.0.23/src/security/ServerOptions.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/ServerOptions.cc 2018-03-08 02:16:46.000000000 +1300 @@ -44,11 +44,8 @@ staticContextSessionId = old.staticContextSessionId; generateHostCertificates = old.generateHostCertificates; - signingCert = old.signingCert; - signPkey = old.signPkey; - certsToChain = old.certsToChain; - untrustedSigningCert = old.untrustedSigningCert; - untrustedSignPkey = old.untrustedSignPkey; + signingCa = old.signingCa; + untrustedSigningCa = old.untrustedSigningCa; dynamicCertMemCacheSize = old.dynamicCertMemCacheSize; } return *this; @@ -189,6 +186,27 @@ return ctx; } +void +Security::ServerOptions::initServerContexts(AnyP::PortCfg &port) +{ + const char *portType = AnyP::ProtocolType_str[port.transport.protocol]; + for (auto &keyData : certs) { + keyData.loadFromFiles(port, portType); + } + + if (generateHostCertificates) { + createSigningContexts(port); + } + + if (!certs.empty() && !createStaticServerContext(port)) { + char buf[128]; + fatalf("%s_port %s initialization error", portType, port.s.toUrl(buf, sizeof(buf))); + } + + // if generate-host-certificates=off and certs is empty, no contexts may be created. + // features depending on contexts do their own checks and error messages later. +} + bool Security::ServerOptions::createStaticServerContext(AnyP::PortCfg &port) { @@ -196,10 +214,61 @@ Security::ContextPointer t(createBlankContext()); if (t) { + #if USE_OPENSSL - if (!Ssl::InitServerContext(t, port)) + if (certs.size() > 1) { + // NOTE: calling SSL_CTX_use_certificate() repeatedly _replaces_ the previous cert details. + // so we cannot use it and support multiple server certificates with OpenSSL. + debugs(83, DBG_CRITICAL, "ERROR: OpenSSL does not support multiple server certificates. Ignoring addional cert= parameters."); + } + + const auto &keys = certs.front(); + + if (!SSL_CTX_use_certificate(t.get(), keys.cert.get())) { + const auto x = ERR_get_error(); + debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS certificate '" << keys.certFile << "': " << Security::ErrorString(x)); return false; + } + + if (!SSL_CTX_use_PrivateKey(t.get(), keys.pkey.get())) { + const auto x = ERR_get_error(); + debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS private key '" << keys.privateKeyFile << "': " << Security::ErrorString(x)); + return false; + } + + for (auto cert : keys.chain) { + if (SSL_CTX_add_extra_chain_cert(t.get(), cert.get())) { + // increase the certificate lock + X509_up_ref(cert.get()); + } else { + const auto error = ERR_get_error(); + debugs(83, DBG_IMPORTANT, "WARNING: can not add certificate to SSL context chain: " << Security::ErrorString(error)); + } + } + +#elif USE_GNUTLS + for (auto &keys : certs) { + gnutls_x509_crt_t crt = keys.cert.get(); + gnutls_x509_privkey_t xkey = keys.pkey.get(); + const auto x = gnutls_certificate_set_x509_key(t.get(), &crt, 1, xkey); + if (x != GNUTLS_E_SUCCESS) { + SBuf whichFile = keys.certFile; + if (keys.certFile != keys.privateKeyFile) { + whichFile.appendf(" and "); + whichFile.append(keys.privateKeyFile); + } + debugs(83, DBG_CRITICAL, "ERROR: Failed to initialize server context with keys from " << whichFile << ": " << Security::ErrorString(x)); + return false; + } + // XXX: add cert chain to the context + } #endif + + if (!updateContextConfig(t)) { + debugs(83, DBG_CRITICAL, "ERROR: Configuring static TLS context"); + return false; + } + if (!loadClientCaFile()) return false; } @@ -209,39 +278,40 @@ } void -Security::ServerOptions::createSigningContexts(AnyP::PortCfg &port) +Security::ServerOptions::createSigningContexts(const AnyP::PortCfg &port) { - const char *portType = AnyP::ProtocolType_str[port.transport.protocol]; - if (!certs.empty()) { -#if USE_OPENSSL - Security::KeyData &keys = certs.front(); - Ssl::readCertChainAndPrivateKeyFromFiles(signingCert, signPkey, certsToChain, keys.certFile.c_str(), keys.privateKeyFile.c_str()); -#else - char buf[128]; - fatalf("Directive '%s_port %s' requires --with-openssl.", portType, port.s.toUrl(buf, sizeof(buf))); -#endif - } + // For signing we do not have a pre-initialized context object. Instead + // contexts are generated as needed. This method initializes the cert + // and key pointers used to sign those contexts later. - if (!signingCert) { + signingCa = certs.front(); + + const char *portType = AnyP::ProtocolType_str[port.transport.protocol]; + if (!signingCa.cert) { char buf[128]; - fatalf("No valid signing SSL certificate configured for %s_port %s", portType, port.s.toUrl(buf, sizeof(buf))); + // XXX: we never actually checked that the cert is capable of signing! + fatalf("No valid signing certificate configured for %s_port %s", portType, port.s.toUrl(buf, sizeof(buf))); } - if (!signPkey) - debugs(3, DBG_IMPORTANT, "No SSL private key configured for " << portType << "_port " << port.s); + if (!signingCa.pkey) + debugs(3, DBG_IMPORTANT, "No TLS private key configured for " << portType << "_port " << port.s); #if USE_OPENSSL - Ssl::generateUntrustedCert(untrustedSigningCert, untrustedSignPkey, signingCert, signPkey); + Ssl::generateUntrustedCert(untrustedSigningCa.cert, untrustedSigningCa.pkey, signingCa.cert, signingCa.pkey); +#elif USE_GNUTLS + // TODO: implement for GnuTLS. Just a warning for now since generate is implicitly on for all crypto builds. + signingCa.cert.reset(); + signingCa.pkey.reset(); + debugs(83, DBG_CRITICAL, "WARNING: Dynamic TLS certificate generation requires --with-openssl."); + return; +#else + debugs(83, DBG_CRITICAL, "ERROR: Dynamic TLS certificate generation requires --with-openssl."); + return; #endif - if (!untrustedSigningCert) { + if (!untrustedSigningCa.cert) { char buf[128]; - fatalf("Unable to generate signing SSL certificate for untrusted sites for %s_port %s", portType, port.s.toUrl(buf, sizeof(buf))); - } - - if (!createStaticServerContext(port)) { - char buf[128]; - fatalf("%s_port %s initialization error", portType, port.s.toUrl(buf, sizeof(buf))); + fatalf("Unable to generate signing certificate for untrusted sites for %s_port %s", portType, port.s.toUrl(buf, sizeof(buf))); } } diff -u -r -N squid-4.0.23/src/security/ServerOptions.h squid-4.0.24/src/security/ServerOptions.h --- squid-4.0.23/src/security/ServerOptions.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/security/ServerOptions.h 2018-03-08 02:16:46.000000000 +1300 @@ -41,14 +41,9 @@ virtual Security::ContextPointer createBlankContext() const; virtual void dumpCfg(Packable *, const char *pfx) const; - /// generate a security server-context from these configured options - /// the resulting context is stored in staticContext - /// \returns true if a context could be created - bool createStaticServerContext(AnyP::PortCfg &); - - /// initialize contexts for signing dynamic TLS certificates (if needed) - /// the resulting context is stored in signingCert, signPKey, untrustedSigningCert, untrustedSignPKey - void createSigningContexts(AnyP::PortCfg &); + /// initialize all server contexts as-needed and load PEM files. + /// if none can be created this may do nothing. + void initServerContexts(AnyP::PortCfg &); /// update the given TLS security context using squid.conf settings bool updateContextConfig(Security::ContextPointer &); @@ -70,13 +65,21 @@ Security::ContextPointer staticContext; SBuf staticContextSessionId; ///< "session id context" for staticContext +#if USE_OPENSSL + bool generateHostCertificates = true; ///< dynamically make host cert +#elif USE_GNUTLS + // TODO: GnuTLS does implement TLS server connections so the cert + // generate vs static choice can be reached in the code now. + // But this feature is not fully working implemented so must not + // be enabled by default for production installations. + bool generateHostCertificates = false; ///< dynamically make host cert +#else + // same as OpenSSL so config errors show up easily bool generateHostCertificates = true; ///< dynamically make host cert +#endif - Security::CertPointer signingCert; ///< x509 certificate for signing generated certificates - Security::PrivateKeyPointer signPkey; ///< private key for signing generated certificates - Security::CertList certsToChain; ///< x509 certificates to send with the generated cert - Security::CertPointer untrustedSigningCert; ///< x509 certificate for signing untrusted generated certificates - Security::PrivateKeyPointer untrustedSignPkey; ///< private key for signing untrusted generated certificates + Security::KeyData signingCa; ///< x509 certificate and key for signing generated certificates + Security::KeyData untrustedSigningCa; ///< x509 certificate and key for signing untrusted generated certificates /// max size of generated certificates memory cache (4 MB default) size_t dynamicCertMemCacheSize = 4*1024*1024; @@ -85,6 +88,15 @@ bool loadClientCaFile(); void loadDhParams(); + /// generate a security server-context from these configured options + /// the resulting context is stored in staticContext + /// \returns true if a context could be created + bool createStaticServerContext(AnyP::PortCfg &); + + /// initialize contexts for signing dynamic TLS certificates (if needed) + /// the resulting keys are stored in signingCa and untrustedSigningCa + void createSigningContexts(const AnyP::PortCfg &); + private: SBuf clientCaFile; ///< name of file to load client CAs from #if USE_OPENSSL diff -u -r -N squid-4.0.23/src/servers/Http1Server.cc squid-4.0.24/src/servers/Http1Server.cc --- squid-4.0.23/src/servers/Http1Server.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/servers/Http1Server.cc 2018-03-08 02:16:46.000000000 +1300 @@ -42,14 +42,12 @@ { ConnStateData::start(); -#if USE_OPENSSL // XXX: Until we create an HttpsServer class, use this hack to allow old // client_side.cc code to manipulate ConnStateData object directly if (isHttpsServer) { postHttpsAccept(); return; } -#endif typedef CommCbMemFunT TimeoutDialer; AsyncCall::Pointer timeoutCall = JobCallback(33, 5, diff -u -r -N squid-4.0.23/src/ssl/bio.h squid-4.0.24/src/ssl/bio.h --- squid-4.0.23/src/ssl/bio.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ssl/bio.h 2018-03-08 02:16:46.000000000 +1300 @@ -13,7 +13,9 @@ #include "FadingCounter.h" #include "fd.h" +#include "MemBuf.h" #include "security/Handshake.h" +#include "ssl/support.h" #include #include diff -u -r -N squid-4.0.23/src/ssl/support.cc squid-4.0.24/src/ssl/support.cc --- squid-4.0.23/src/ssl/support.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ssl/support.cc 2018-03-08 02:16:46.000000000 +1300 @@ -59,15 +59,14 @@ \ingroup ServerProtocolSSLAPI */ -/// \ingroup ServerProtocolSSLInternal -static int -ssl_ask_password_cb(char *buf, int size, int rwflag, void *userdata) +int +Ssl::AskPasswordCb(char *buf, int size, int rwflag, void *userdata) { FILE *in; int len = 0; char cmdline[1024]; - snprintf(cmdline, sizeof(cmdline), "\"%s\" \"%s\"", Config.Program.ssl_password, (const char *)userdata); + snprintf(cmdline, sizeof(cmdline), "\"%s\" \"%s\"", ::Config.Program.ssl_password, (const char *)userdata); in = popen(cmdline, "r"); if (fgets(buf, size, in)) @@ -89,7 +88,7 @@ ssl_ask_password(SSL_CTX * context, const char * prompt) { if (Config.Program.ssl_password) { - SSL_CTX_set_default_passwd_cb(context, ssl_ask_password_cb); + SSL_CTX_set_default_passwd_cb(context, Ssl::AskPasswordCb); SSL_CTX_set_default_passwd_cb_userdata(context, (void *)prompt); } } @@ -512,27 +511,6 @@ if (!ctx) return false; - if (!SSL_CTX_use_certificate(ctx.get(), port.secure.signingCert.get())) { - const int ssl_error = ERR_get_error(); - const auto &keys = port.secure.certs.front(); - debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS certificate '" << keys.certFile << "': " << Security::ErrorString(ssl_error)); - return false; - } - - if (!SSL_CTX_use_PrivateKey(ctx.get(), port.secure.signPkey.get())) { - const int ssl_error = ERR_get_error(); - const auto &keys = port.secure.certs.front(); - debugs(83, DBG_CRITICAL, "ERROR: Failed to acquire TLS private key '" << keys.privateKeyFile << "': " << Security::ErrorString(ssl_error)); - return false; - } - - Ssl::addChainToSslContext(ctx, port.secure.certsToChain); - - if (!port.secure.updateContextConfig(ctx)) { - debugs(83, DBG_CRITICAL, "ERROR: Configuring static SSL context"); - return false; - } - return true; } @@ -837,7 +815,7 @@ { assert(ctx); // Add signing certificate to the certificates chain - X509 *signingCert = options.signingCert.get(); + X509 *signingCert = options.signingCa.cert.get(); if (SSL_CTX_add_extra_chain_cert(ctx.get(), signingCert)) { // increase the certificate lock X509_up_ref(signingCert); @@ -845,7 +823,16 @@ const int ssl_error = ERR_get_error(); debugs(33, DBG_IMPORTANT, "WARNING: can not add signing certificate to SSL context chain: " << Security::ErrorString(ssl_error)); } - Ssl::addChainToSslContext(ctx, options.certsToChain); + + for (auto cert : options.signingCa.chain) { + if (SSL_CTX_add_extra_chain_cert(ctx.get(), cert.get())) { + // increase the certificate lock + X509_up_ref(cert.get()); + } else { + const auto error = ERR_get_error(); + debugs(83, DBG_IMPORTANT, "WARNING: can not add certificate to SSL dynamic context chain: " << Security::ErrorString(error)); + } + } } void @@ -943,23 +930,6 @@ #endif } -void -Ssl::addChainToSslContext(Security::ContextPointer &ctx, Security::CertList &chain) -{ - if (chain.empty()) - return; - - for (auto cert : chain) { - if (SSL_CTX_add_extra_chain_cert(ctx.get(), cert.get())) { - // increase the certificate lock - X509_up_ref(cert.get()); - } else { - const int ssl_error = ERR_get_error(); - debugs(83, DBG_IMPORTANT, "WARNING: can not add certificate to SSL context chain: " << Security::ErrorString(ssl_error)); - } - } -} - static const char * hasAuthorityInfoAccessCaIssuers(X509 *cert) { @@ -1236,65 +1206,6 @@ } } -/** - \ingroup ServerProtocolSSLInternal - * Read certificate from file. - * See also: static readSslX509Certificate function, gadgets.cc file - */ -static X509 * readSslX509CertificatesChain(char const * certFilename, Security::CertList &chain) -{ - if (!certFilename) - return NULL; - Ssl::BIO_Pointer bio(BIO_new(BIO_s_file())); - if (!bio) - return NULL; - if (!BIO_read_filename(bio.get(), certFilename)) - return NULL; - X509 *certificate = PEM_read_bio_X509(bio.get(), NULL, NULL, NULL); - - if (certificate) { - - if (X509_check_issued(certificate, certificate) == X509_V_OK) - debugs(83, 5, "Certificate is self-signed, will not be chained"); - else { - // and add to the chain any other certificate exist in the file - while (X509 *ca = PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr)) - chain.emplace_front(Security::CertPointer(ca)); - } - } - - return certificate; -} - -void -Ssl::readCertChainAndPrivateKeyFromFiles(Security::CertPointer & cert, Security::PrivateKeyPointer & pkey, Security::CertList &chain, char const * certFilename, char const * keyFilename) -{ - if (keyFilename == NULL) - keyFilename = certFilename; - - if (certFilename == NULL) - certFilename = keyFilename; - - debugs(83, DBG_IMPORTANT, "Using certificate in " << certFilename); - - // XXX: ssl_ask_password_cb needs SSL_CTX_set_default_passwd_cb_userdata() - // so this may not fully work iff Config.Program.ssl_password is set. - pem_password_cb *cb = ::Config.Program.ssl_password ? &ssl_ask_password_cb : NULL; - Ssl::ReadPrivateKeyFromFile(keyFilename, pkey, cb); - cert.resetWithoutLocking(readSslX509CertificatesChain(certFilename, chain)); - if (!cert) { - debugs(83, DBG_IMPORTANT, "WARNING: missing cert in '" << certFilename << "'"); - } else if (!pkey) { - debugs(83, DBG_IMPORTANT, "WARNING: missing private key in '" << keyFilename << "'"); - } else if (!X509_check_private_key(cert.get(), pkey.get())) { - debugs(83, DBG_IMPORTANT, "WARNING: X509_check_private_key() failed to verify signing cert"); - } else - return; // everything is okay - - pkey.reset(); - cert.reset(); -} - bool Ssl::generateUntrustedCert(Security::CertPointer &untrustedCert, Security::PrivateKeyPointer &untrustedPkey, Security::CertPointer const &cert, Security::PrivateKeyPointer const & pkey) { // Generate the self-signed certificate, using a hard-coded subject prefix diff -u -r -N squid-4.0.23/src/ssl/support.h squid-4.0.24/src/ssl/support.h --- squid-4.0.23/src/ssl/support.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/ssl/support.h 2018-03-08 02:16:46.000000000 +1300 @@ -65,6 +65,11 @@ namespace Ssl { + +/// callback for receiving password to access password secured PEM files +/// XXX: Requires SSL_CTX_set_default_passwd_cb_userdata()! +int AskPasswordCb(char *buf, int size, int rwflag, void *userdata); + /// initialize the SSL library global state. /// call before generating any SSL context void Initialize(); @@ -266,27 +271,12 @@ /** \ingroup ServerProtocolSSLAPI - * Adds the certificates in certList to the certificate chain of the SSL context - */ -void addChainToSslContext(Security::ContextPointer &, Security::CertList &); - -/** - \ingroup ServerProtocolSSLAPI * Configures sslContext to use squid untrusted certificates internal list * to complete certificate chains when verifies SSL servers certificates. */ void useSquidUntrusted(SSL_CTX *sslContext); /** - \ingroup ServerProtocolSSLAPI - * Read certificate, private key and any certificates which must be chained from files. - * See also: Ssl::readCertAndPrivateKeyFromFiles function, defined in gadgets.h - * \param certFilename name of file with certificate and certificates which must be chainned. - * \param keyFilename name of file with private key. - */ -void readCertChainAndPrivateKeyFromFiles(Security::CertPointer & cert, Security::PrivateKeyPointer & pkey, Security::CertList &chain, char const * certFilename, char const * keyFilename); - -/** \ingroup ServerProtocolSSLAPI * Iterates over the X509 common and alternate names and to see if matches with given data * using the check_func. diff -u -r -N squid-4.0.23/src/store/Controlled.h squid-4.0.24/src/store/Controlled.h --- squid-4.0.23/src/store/Controlled.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Controlled.h 2018-03-08 02:16:46.000000000 +1300 @@ -18,6 +18,12 @@ class Controlled: public Storage { public: + /// \returns a possibly unlocked/unregistered stored entry with key (or nil) + /// The returned entry might not match the caller's Store ID or method. The + /// caller must abandon()/release() the entry or register it with Root(). + /// This method must not trigger slow I/O operations (e.g., disk swap in). + virtual StoreEntry *get(const cache_key *) = 0; + /// somebody needs this entry (many cache replacement policies need to know) virtual void reference(StoreEntry &e) = 0; @@ -28,15 +34,15 @@ /// make stored metadata and HTTP headers the same as in the given entry virtual void updateHeaders(StoreEntry *) {} - /// If this storage cannot cache collapsed entries, return false. + /// If Transients entry cannot be attached to this storage, return false. /// If the entry is not found, return false. Otherwise, return true after - /// tying the entry to this cache and setting inSync to updateCollapsed(). - virtual bool anchorCollapsed(StoreEntry &, bool &/*inSync*/) { return false; } + /// tying the entry to this cache and setting inSync to updateAnchored(). + virtual bool anchorToCache(StoreEntry &, bool &/*inSync*/) { return false; } - /// Update a local collapsed entry with fresh info from this cache (if any). - /// Return true iff the cache supports collapsed entries and - /// the given local collapsed entry is now in sync with this storage. - virtual bool updateCollapsed(StoreEntry &) { return false; } + /// Update a local Transients entry with fresh info from this cache (if any). + /// Return true iff the cache supports Transients entries and + /// the given local Transients entry is now in sync with this storage. + virtual bool updateAnchored(StoreEntry &) { return false; } }; } // namespace Store diff -u -r -N squid-4.0.23/src/store/Controller.cc squid-4.0.24/src/store/Controller.cc --- squid-4.0.23/src/store/Controller.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Controller.cc 2018-03-08 02:16:46.000000000 +1300 @@ -233,6 +233,7 @@ return result; } +/// update reference counters of the recently touched entry void Store::Controller::referenceBusy(StoreEntry &e) { @@ -242,7 +243,7 @@ /* Notify the fs that we're referencing this object again */ - if (e.swap_dirn > -1) + if (e.hasDisk()) swapDir->reference(e); // Notify the memory cache that we're referencing this object again @@ -256,6 +257,8 @@ } } +/// dereference()s an idle entry +/// \returns false if and only if the entry should be deleted bool Store::Controller::dereferenceIdle(StoreEntry &e, bool wantsLocalMemory) { @@ -267,7 +270,7 @@ /* Notify the fs that we're not referencing this object any more */ - if (e.swap_filen > -1) + if (e.hasDisk()) keepInStoreTable = swapDir->dereference(e) || keepInStoreTable; // Notify the memory cache that we're not referencing this object any more @@ -286,29 +289,107 @@ return keepInStoreTable; } +bool +Store::Controller::markedForDeletion(const cache_key *key) const +{ + // assuming a public key, checking Transients should cover all cases. + return transients && transients->markedForDeletion(key); +} + +bool +Store::Controller::markedForDeletionAndAbandoned(const StoreEntry &e) const +{ + // The opposite check order could miss a reader that has arrived after the + // !readers() and before the markedForDeletion() check. + return markedForDeletion(reinterpret_cast(e.key)) && + transients && !transients->readers(e); +} + +bool +Store::Controller::hasReadableDiskEntry(const StoreEntry &e) const +{ + return swapDir->hasReadableEntry(e); +} + StoreEntry * -Store::Controller::get(const cache_key *key) +Store::Controller::find(const cache_key *key) { - if (StoreEntry *e = find(key)) { - // this is not very precise: some get()s are not initiated by clients - e->touch(); - referenceBusy(*e); - return e; + if (const auto entry = peek(key)) { + try { + if (!entry->key) + allowSharing(*entry, key); + checkTransients(*entry); + entry->touch(); + referenceBusy(*entry); + return entry; + } catch (const std::exception &ex) { + debugs(20, 2, "failed with " << *entry << ": " << ex.what()); + entry->release(true); + // fall through + } } return NULL; } -/// Internal method to implements the guts of the Store::get() API: -/// returns an in-transit or cached object with a given key, if any. +/// indexes and adds SMP-tracking for an ephemeral peek() result +void +Store::Controller::allowSharing(StoreEntry &entry, const cache_key *key) +{ + // TODO: refactor to throw on anchorToCache() inSync errors! + + // anchorToCache() below and many find() callers expect a registered entry + addReading(&entry, key); + + if (entry.hasTransients()) { + bool inSync = false; + const bool found = anchorToCache(entry, inSync); + if (found && !inSync) + throw TexcHere("cannot sync"); + } +} + StoreEntry * -Store::Controller::find(const cache_key *key) +Store::Controller::findCallback(const cache_key *key) { - debugs(20, 3, storeKeyText(key)); + // We could check for mem_obj presence (and more), moving and merging some + // of the duplicated neighborsUdpAck() and neighborsHtcpReply() code here, + // but that would mean polluting Store with HTCP/ICP code. Instead, we + // should encapsulate callback-related data in a protocol-neutral MemObject + // member or use an HTCP/ICP-specific index rather than store_table. + return peekAtLocal(key); +} +/// \returns either an existing local reusable StoreEntry object or nil +/// To treat remotely marked entries specially, +/// callers ought to check markedForDeletion() first! +StoreEntry * +Store::Controller::peekAtLocal(const cache_key *key) +{ if (StoreEntry *e = static_cast(hash_lookup(store_table, key))) { + // callers must only search for public entries + assert(!EBIT_TEST(e->flags, KEY_PRIVATE)); + assert(e->publicKey()); + checkTransients(*e); + // TODO: ignore and maybe handleIdleEntry() unlocked intransit entries // because their backing store slot may be gone already. - debugs(20, 3, HERE << "got in-transit entry: " << *e); + return e; + } + return nullptr; +} + +StoreEntry * +Store::Controller::peek(const cache_key *key) +{ + debugs(20, 3, storeKeyText(key)); + + if (markedForDeletion(key)) { + debugs(20, 3, "ignoring marked in-transit " << storeKeyText(key)); + return nullptr; + } + + if (StoreEntry *e = peekAtLocal(key)) { + debugs(20, 3, "got local in-transit entry: " << *e); return e; } @@ -316,13 +397,7 @@ if (transients) { if (StoreEntry *e = transients->get(key)) { debugs(20, 3, "got shared in-transit entry: " << *e); - bool inSync = false; - const bool found = anchorCollapsed(*e, inSync); - if (!found || inSync) - return e; - assert(!e->locked()); // ensure release will destroyStoreEntry() - e->release(); // do not let others into the same trap - return NULL; + return e; } } @@ -344,6 +419,18 @@ return nullptr; } +bool +Store::Controller::transientsReader(const StoreEntry &e) const +{ + return transients && e.hasTransients() && transients->isReader(e); +} + +bool +Store::Controller::transientsWriter(const StoreEntry &e) const +{ + return transients && e.hasTransients() && transients->isWriter(e); +} + int64_t Store::Controller::accumulateMore(StoreEntry &entry) const { @@ -351,23 +438,87 @@ // The memory cache should not influence for-swapout accumulation decision. } +// Must be called from StoreEntry::release() or releaseRequest() because +// those methods currently manage local indexing of StoreEntry objects. +// TODO: Replace StoreEntry::release*() with Root().evictCached(). +void +Store::Controller::evictCached(StoreEntry &e) +{ + debugs(20, 7, e); + if (transients) + transients->evictCached(e); + memoryEvictCached(e); + if (swapDir) + swapDir->evictCached(e); +} + void -Store::Controller::markForUnlink(StoreEntry &e) +Store::Controller::evictIfFound(const cache_key *key) +{ + debugs(20, 7, storeKeyText(key)); + + if (StoreEntry *entry = peekAtLocal(key)) { + debugs(20, 5, "marking local in-transit " << *entry); + entry->release(true); + return; + } + + if (memStore) + memStore->evictIfFound(key); + if (swapDir) + swapDir->evictIfFound(key); + if (transients) + transients->evictIfFound(key); +} + +/// whether the memory cache is allowed to store that many additional pages +bool +Store::Controller::memoryCacheHasSpaceFor(const int pagesRequired) const { - if (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0) - transients->markForUnlink(e); - if (memStore && e.mem_obj && e.mem_obj->memCache.index >= 0) - memStore->markForUnlink(e); - if (swapDir && e.swap_filen >= 0) - swapDir->markForUnlink(e); + // XXX: We count mem_nodes but may free shared memory pages instead. + const auto fits = mem_node::InUseCount() + pagesRequired <= store_pages_max; + debugs(20, 7, fits << ": " << mem_node::InUseCount() << '+' << pagesRequired << '?' << store_pages_max); + return fits; } void -Store::Controller::unlink(StoreEntry &e) +Store::Controller::freeMemorySpace(const int bytesRequired) { - memoryUnlink(e); - if (swapDir && e.swap_filen >= 0) - swapDir->unlink(e); + const auto pagesRequired = (bytesRequired + SM_PAGE_SIZE-1) / SM_PAGE_SIZE; + + if (memoryCacheHasSpaceFor(pagesRequired)) + return; + + // XXX: When store_pages_max is smaller than pagesRequired, we should not + // look for more space (but we do because we want to abandon idle entries?). + + // limit our performance impact to one walk per second + static time_t lastWalk = 0; + if (lastWalk == squid_curtime) + return; + lastWalk = squid_curtime; + + debugs(20, 2, "need " << pagesRequired << " pages"); + + // let abandon()/handleIdleEntry() know about the impeding memory shortage + memoryPagesDebt_ = pagesRequired; + + // XXX: SMP-unaware: Walkers should iterate memory cache, not store_table. + // XXX: Limit iterations by time, not arbitrary count. + const auto walker = mem_policy->PurgeInit(mem_policy, 100000); + int removed = 0; + while (const auto entry = walker->Next(walker)) { + // Abandoned memory cache entries are purged during memory shortage. + entry->abandon(__FUNCTION__); // may delete entry + ++removed; + + if (memoryCacheHasSpaceFor(pagesRequired)) + break; + } + // TODO: Move to RemovalPolicyWalker::Done() that has more/better details. + debugs(20, 3, "removed " << removed << " out of " << hot_obj_count << " memory-cached entries"); + walker->Done(walker); + memoryPagesDebt_ = 0; } // move this into [non-shared] memory cache class when we have one @@ -404,13 +555,17 @@ e.trimMemory(preserveSwappable); } +/// removes the entry from the memory cache +/// XXX: Dangerous side effect: Unlocked entries lose their mem_obj. void -Store::Controller::memoryUnlink(StoreEntry &e) +Store::Controller::memoryEvictCached(StoreEntry &e) { + // TODO: Untangle memory caching from mem_obj. if (memStore) - memStore->unlink(e); + memStore->evictCached(e); else // TODO: move into [non-shared] memory cache class when we have one - e.destroyMemObject(); + if (!e.locked()) + e.destroyMemObject(); } void @@ -422,37 +577,38 @@ } void -Store::Controller::transientsAbandon(StoreEntry &e) +Store::Controller::stopSharing(StoreEntry &e) { - if (transients) { - assert(e.mem_obj); - if (e.mem_obj->xitTable.index >= 0) - transients->abandon(e); - } + // Marking the transients entry is sufficient to prevent new readers from + // starting to wait for `e` updates and to inform the current readers (and, + // hence, Broadcast() recipients) about the underlying Store problems. + if (transients && e.hasTransients()) + transients->evictCached(e); } void Store::Controller::transientsCompleteWriting(StoreEntry &e) { - if (transients) { - assert(e.mem_obj); - if (e.mem_obj->xitTable.index >= 0) - transients->completeWriting(e); - } + // transients->isWriter(e) is false if `e` is writing to its second store + // after finishing writing to its first store: At the end of the first swap + // out, the transients writer becomes a reader and (XXX) we never switch + // back to writing, even if we start swapping out again (to another store). + if (transients && e.hasTransients() && transients->isWriter(e)) + transients->completeWriting(e); } int Store::Controller::transientReaders(const StoreEntry &e) const { - return (transients && e.mem_obj && e.mem_obj->xitTable.index >= 0) ? + return (transients && e.hasTransients()) ? transients->readers(e) : 0; } void -Store::Controller::transientsDisconnect(MemObject &mem_obj) +Store::Controller::transientsDisconnect(StoreEntry &e) { if (transients) - transients->disconnect(mem_obj); + transients->disconnect(e); } void @@ -470,7 +626,7 @@ } else { keepInLocalMemory = keepForLocalMemoryCache(e) && // in good shape and // the local memory cache is not overflowing - (mem_node::InUseCount() <= store_pages_max); + memoryCacheHasSpaceFor(memoryPagesDebt_); } // An idle, unlocked entry that only belongs to a SwapDir which controls @@ -487,9 +643,18 @@ if (keepInLocalMemory) { e.setMemStatus(IN_MEMORY); e.mem_obj->unlinkRequest(); - } else { - e.purgeMem(); // may free e + return; } + + // We know the in-memory data will be gone. Get rid of the entire entry if + // it has nothing worth preserving on disk either. + if (!e.swappedOut()) { + e.release(); // deletes e + return; + } + + memoryEvictCached(e); // may already be gone + // and keep the entry in store_table for its on-disk data } void @@ -509,20 +674,41 @@ if (memStore && old->mem_status == IN_MEMORY && !EBIT_TEST(old->flags, ENTRY_SPECIAL)) memStore->updateHeaders(old); - if (old->swap_dirn > -1) + if (old->hasDisk()) swapDir->updateHeaders(old); } -void +bool Store::Controller::allowCollapsing(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod) { const KeyScope keyScope = reqFlags.refresh ? ksRevalidation : ksDefault; - e->makePublic(keyScope); // this is needed for both local and SMP collapsing + if (e->makePublic(keyScope)) { // this is needed for both local and SMP collapsing + debugs(20, 3, "may " << (transients && e->hasTransients() ? + "SMP-" : "locally-") << "collapse " << *e); + return true; + } + return false; +} + +void +Store::Controller::addReading(StoreEntry *e, const cache_key *key) +{ if (transients) - transients->startWriting(e, reqFlags, reqMethod); - debugs(20, 3, "may " << (transients && e->mem_obj->xitTable.index >= 0 ? - "SMP-" : "locally-") << "collapse " << *e); + transients->monitorIo(e, key, Store::ioReading); + e->hashInsert(key); +} + +void +Store::Controller::addWriting(StoreEntry *e, const cache_key *key) +{ + assert(e); + if (EBIT_TEST(e->flags, ENTRY_SPECIAL)) + return; // constant memory-resident entries do not need transients + + if (transients) + transients->monitorIo(e, key, Store::ioWriting); + // else: non-SMP configurations do not need transients } void @@ -531,34 +717,66 @@ assert(transients); StoreEntry *collapsed = transients->findCollapsed(xitIndex); - if (!collapsed) { // the entry is no longer locally active, ignore update + if (!collapsed) { // the entry is no longer active, ignore update debugs(20, 7, "not SMP-syncing not-transient " << xitIndex); return; } + + if (!collapsed->locked()) { + debugs(20, 3, "skipping (and may destroy) unlocked " << *collapsed); + handleIdleEntry(*collapsed); + return; + } + assert(collapsed->mem_obj); - assert(collapsed->mem_obj->smpCollapsed); + + if (EBIT_TEST(collapsed->flags, ENTRY_ABORTED)) { + debugs(20, 3, "skipping already aborted " << *collapsed); + return; + } debugs(20, 7, "syncing " << *collapsed); - bool abandoned = transients->abandoned(*collapsed); + bool abortedByWriter = false; + bool waitingToBeFreed = false; + transients->status(*collapsed, abortedByWriter, waitingToBeFreed); + + if (waitingToBeFreed) { + debugs(20, 3, "will release " << *collapsed << " due to waitingToBeFreed"); + collapsed->release(true); // may already be marked + } + + if (transients->isWriter(*collapsed)) + return; // readers can only change our waitingToBeFreed flag + + assert(transients->isReader(*collapsed)); + + if (abortedByWriter) { + debugs(20, 3, "aborting " << *collapsed << " because its writer has aborted"); + collapsed->abort(); + return; + } + bool found = false; bool inSync = false; if (memStore && collapsed->mem_obj->memCache.io == MemObject::ioDone) { found = true; inSync = true; debugs(20, 7, "fully mem-loaded " << *collapsed); - } else if (memStore && collapsed->mem_obj->memCache.index >= 0) { + } else if (memStore && collapsed->hasMemStore()) { found = true; - inSync = memStore->updateCollapsed(*collapsed); - } else if (swapDir && collapsed->swap_filen >= 0) { + inSync = memStore->updateAnchored(*collapsed); + // TODO: handle entries attached to both memory and disk + } else if (swapDir && collapsed->hasDisk()) { found = true; - inSync = swapDir->updateCollapsed(*collapsed); + inSync = swapDir->updateAnchored(*collapsed); } else { - found = anchorCollapsed(*collapsed, inSync); + found = anchorToCache(*collapsed, inSync); } - if (abandoned && collapsed->store_status == STORE_PENDING) { - debugs(20, 3, "aborting abandoned but STORE_PENDING " << *collapsed); + if (waitingToBeFreed && !found) { + debugs(20, 3, "aborting unattached " << *collapsed << + " because it was marked for deletion before we could attach it"); collapsed->abort(); return; } @@ -566,40 +784,43 @@ if (inSync) { debugs(20, 5, "synced " << *collapsed); collapsed->invokeHandlers(); - } else if (found) { // unrecoverable problem syncing this entry + return; + } + + if (found) { // unrecoverable problem syncing this entry debugs(20, 3, "aborting unsyncable " << *collapsed); collapsed->abort(); - } else { // the entry is still not in one of the caches - debugs(20, 7, "waiting " << *collapsed); + return; } + + // the entry is still not in one of the caches + debugs(20, 7, "waiting " << *collapsed); } -/// Called for in-transit entries that are not yet anchored to a cache. +/// Called for Transients entries that are not yet anchored to a cache. /// For cached entries, return true after synchronizing them with their cache /// (making inSync true on success). For not-yet-cached entries, return false. bool -Store::Controller::anchorCollapsed(StoreEntry &collapsed, bool &inSync) +Store::Controller::anchorToCache(StoreEntry &entry, bool &inSync) { - // this method is designed to work with collapsed transients only - assert(collapsed.mem_obj); - assert(collapsed.mem_obj->xitTable.index >= 0); - assert(collapsed.mem_obj->smpCollapsed); + assert(entry.hasTransients()); + assert(transientsReader(entry)); - debugs(20, 7, "anchoring " << collapsed); + debugs(20, 7, "anchoring " << entry); bool found = false; if (memStore) - found = memStore->anchorCollapsed(collapsed, inSync); + found = memStore->anchorToCache(entry, inSync); if (!found && swapDir) - found = swapDir->anchorCollapsed(collapsed, inSync); + found = swapDir->anchorToCache(entry, inSync); if (found) { if (inSync) - debugs(20, 7, "anchored " << collapsed); + debugs(20, 7, "anchored " << entry); else - debugs(20, 5, "failed to anchor " << collapsed); + debugs(20, 5, "failed to anchor " << entry); } else { - debugs(20, 7, "skipping not yet cached " << collapsed); + debugs(20, 7, "skipping not yet cached " << entry); } return found; @@ -611,6 +832,14 @@ return memStore || (swapDir && swapDir->smpAware()); } +void +Store::Controller::checkTransients(const StoreEntry &e) const +{ + if (EBIT_TEST(e.flags, ENTRY_SPECIAL)) + return; + assert(!transients || e.hasTransients()); +} + namespace Store { static RefCount TheRoot; } diff -u -r -N squid-4.0.23/src/store/Controller.h squid-4.0.24/src/store/Controller.h --- squid-4.0.23/src/store/Controller.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Controller.h 2018-03-08 02:16:46.000000000 +1300 @@ -28,7 +28,6 @@ /* Storage API */ virtual void create() override; virtual void init() override; - virtual StoreEntry *get(const cache_key *) override; virtual uint64_t maxSize() const override; virtual uint64_t minSize() const override; virtual uint64_t currentSize() const override; @@ -38,11 +37,40 @@ virtual void stat(StoreEntry &) const override; virtual void sync() override; virtual void maintain() override; - virtual void markForUnlink(StoreEntry &) override; - virtual void unlink(StoreEntry &) override; + virtual void evictCached(StoreEntry &) override; + virtual void evictIfFound(const cache_key *) override; virtual int callback() override; virtual bool smpAware() const override; + /// \returns a locally indexed and SMP-tracked matching StoreEntry (or nil) + /// Slower than peek() but does not restrict StoreEntry use and storage. + /// Counts as an entry reference from the removal policy p.o.v. + StoreEntry *find(const cache_key *); + + /// \returns a matching StoreEntry not suitable for long-term use (or nil) + /// Faster than find() but the returned entry may not receive updates, may + /// lack information from some of the Stores, and should not be updated + /// except that purging peek()ed entries is supported. + /// Does not count as an entry reference from the removal policy p.o.v. + StoreEntry *peek(const cache_key *); + + /// \returns matching StoreEntry associated with local ICP/HTCP transaction + /// Warning: The returned StoreEntry is not synced and may be marked for + /// deletion. Use it only for extracting transaction callback details. + /// TODO: Group and return just that callback-related data instead? + StoreEntry *findCallback(const cache_key *); + + /// Whether a transient entry with the given public key exists and (but) was + /// marked for removal some time ago; get(key) returns nil in such cases. + bool markedForDeletion(const cache_key *key) const; + + /// markedForDeletion() with no readers + /// this is one method because the two conditions must be checked in the right order + bool markedForDeletionAndAbandoned(const StoreEntry &) const; + + /// whether there is a disk entry with e.key + bool hasReadableDiskEntry(const StoreEntry &) const; + /// Additional unknown-size entry bytes required by Store in order to /// reduce the risk of selecting the wrong disk cache for the growing entry. int64_t accumulateMore(StoreEntry &) const; @@ -53,14 +81,33 @@ /// called when the entry is no longer needed by any transaction void handleIdleEntry(StoreEntry &); + /// Evict memory cache entries to free at least `spaceRequired` bytes. + /// Should be called via storeGetMemSpace(). + /// Unreliable: Fails if enough victims cannot be found fast enough. + void freeMemorySpace(const int spaceRequired); + /// called to get rid of no longer needed entry data in RAM, if any void memoryOut(StoreEntry &, const bool preserveSwappable); /// update old entry metadata and HTTP headers using a newer entry void updateOnNotModified(StoreEntry *old, const StoreEntry &newer); - /// makes the entry available for collapsing future requests - void allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &); + /// tries to make the entry available for collapsing future requests + bool allowCollapsing(StoreEntry *, const RequestFlags &, const HttpRequestMethod &); + + /// register a being-read StoreEntry (to optimize concurrent cache reads + /// and to receive remote DELETE events) + void addReading(StoreEntry *, const cache_key *); + + /// register a being-written StoreEntry (to support concurrent cache reads + /// and to receive remote DELETE events) + void addWriting(StoreEntry *, const cache_key *); + + /// whether the entry is in "reading from Transients" I/O state + bool transientsReader(const StoreEntry &) const; + + /// whether the entry is in "writing to Transients" I/O state + bool transientsWriter(const StoreEntry &) const; /// marks the entry completed for collapsed requests void transientsCompleteWriting(StoreEntry &); @@ -68,17 +115,14 @@ /// Update local intransit entry after changes made by appending worker. void syncCollapsed(const sfileno); - /// calls Root().transients->abandon() if transients are tracked - void transientsAbandon(StoreEntry &); + /// stop any current (and prevent any future) SMP sharing of the given entry + void stopSharing(StoreEntry &); /// number of the transient entry readers some time ago int transientReaders(const StoreEntry &) const; /// disassociates the entry from the intransit table - void transientsDisconnect(MemObject &); - - /// removes the entry from the memory cache - void memoryUnlink(StoreEntry &); + void transientsDisconnect(StoreEntry &); /// disassociates the entry from the memory cache, preserving cached data void memoryDisconnect(StoreEntry &); @@ -90,14 +134,19 @@ static int store_dirs_rebuilding; private: - /// update reference counters of the recently touched entry + bool memoryCacheHasSpaceFor(const int pagesRequired) const; + void referenceBusy(StoreEntry &e); - /// dereference() an idle entry and return true if the entry should be deleted bool dereferenceIdle(StoreEntry &, bool wantsLocalMemory); - StoreEntry *find(const cache_key *key); + void allowSharing(StoreEntry &, const cache_key *); + StoreEntry *peekAtLocal(const cache_key *); + + void memoryEvictCached(StoreEntry &); + void transientsUnlinkByKeyIfFound(const cache_key *); bool keepForLocalMemoryCache(StoreEntry &e) const; - bool anchorCollapsed(StoreEntry &, bool &inSync); + bool anchorToCache(StoreEntry &e, bool &inSync); + void checkTransients(const StoreEntry &) const; Disks *swapDir; ///< summary view of all disk caches Memory *memStore; ///< memory cache @@ -106,6 +155,9 @@ /// will belong to a memory cache, a disk cache, or will be uncachable /// when the response header comes. Used for SMP collapsed forwarding. Transients *transients; + + /// Hack: Relays page shortage from freeMemorySpace() to handleIdleEntry(). + int memoryPagesDebt_ = 0; }; /// safely access controller singleton diff -u -r -N squid-4.0.23/src/store/Disk.cc squid-4.0.24/src/store/Disk.cc --- squid-4.0.23/src/store/Disk.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Disk.cc 2018-03-08 02:16:46.000000000 +1300 @@ -187,10 +187,10 @@ bool Store::Disk::canLog(StoreEntry const &e)const { - if (e.swap_filen < 0) + if (!e.hasDisk()) return false; - if (e.swap_status != SWAPOUT_DONE) + if (!e.swappedOut()) return false; if (e.swap_file_sz <= 0) diff -u -r -N squid-4.0.23/src/store/Disk.h squid-4.0.24/src/store/Disk.h --- squid-4.0.23/src/store/Disk.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Disk.h 2018-03-08 02:16:46.000000000 +1300 @@ -69,8 +69,13 @@ /// called when the entry is about to forget its association with cache_dir virtual void disconnect(StoreEntry &) {} - /// called when entry swap out is complete - virtual void swappedOut(const StoreEntry &e) = 0; + /// finalize the successful swapout that has been already noticed by Store + virtual void finalizeSwapoutSuccess(const StoreEntry &) = 0; + /// abort the failed swapout that has been already noticed by Store + virtual void finalizeSwapoutFailure(StoreEntry &) = 0; + + /// whether this cache dir has an entry with `e.key` + virtual bool hasReadableEntry(const StoreEntry &e) const = 0; protected: void parseOptions(int reconfiguring); diff -u -r -N squid-4.0.23/src/store/Disks.cc squid-4.0.24/src/store/Disks.cc --- squid-4.0.23/src/store/Disks.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Disks.cc 2018-03-08 02:16:46.000000000 +1300 @@ -486,19 +486,33 @@ } void -Store::Disks::markForUnlink(StoreEntry &e) { - if (e.swap_filen >= 0) - store(e.swap_dirn)->markForUnlink(e); +Store::Disks::evictCached(StoreEntry &e) { + if (e.hasDisk()) { + // TODO: move into Fs::Ufs::UFSSwapDir::evictCached() + if (!EBIT_TEST(e.flags, KEY_PRIVATE)) { + // log before evictCached() below may clear hasDisk() + storeDirSwapLog(&e, SWAP_LOG_DEL); + } + + e.disk().evictCached(e); + return; + } + + if (const auto key = e.publicKey()) + evictIfFound(key); } void -Store::Disks::unlink(StoreEntry &e) { - if (e.swap_filen >= 0) - store(e.swap_dirn)->unlink(e); +Store::Disks::evictIfFound(const cache_key *key) +{ + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) { + if (dir(i).active()) + dir(i).evictIfFound(key); + } } bool -Store::Disks::anchorCollapsed(StoreEntry &collapsed, bool &inSync) +Store::Disks::anchorToCache(StoreEntry &entry, bool &inSync) { if (const int cacheDirs = Config.cacheSwap.n_configured) { // ask each cache_dir until the entry is found; use static starting @@ -511,23 +525,23 @@ if (!sd.active()) continue; - if (sd.anchorCollapsed(collapsed, inSync)) { - debugs(20, 3, "cache_dir " << idx << " anchors " << collapsed); + if (sd.anchorToCache(entry, inSync)) { + debugs(20, 3, "cache_dir " << idx << " anchors " << entry); return true; } } } debugs(20, 4, "none of " << Config.cacheSwap.n_configured << - " cache_dirs have " << collapsed); + " cache_dirs have " << entry); return false; } bool -Store::Disks::updateCollapsed(StoreEntry &collapsed) +Store::Disks::updateAnchored(StoreEntry &entry) { - return collapsed.swap_filen >= 0 && - dir(collapsed.swap_dirn).updateCollapsed(collapsed); + return entry.hasDisk() && + dir(entry.swap_dirn).updateAnchored(entry); } bool @@ -543,7 +557,14 @@ return false; } -/* Store::Disks globals that should be converted to use RegisteredRunner */ +bool +Store::Disks::hasReadableEntry(const StoreEntry &e) const +{ + for (int i = 0; i < Config.cacheSwap.n_configured; ++i) + if (dir(i).active() && dir(i).hasReadableEntry(e)) + return true; + return false; +} void storeDirOpenSwapLogs() @@ -713,7 +734,7 @@ { assert (e); assert(!EBIT_TEST(e->flags, KEY_PRIVATE)); - assert(e->swap_filen >= 0); + assert(e->hasDisk()); /* * icons and such; don't write them to the swap log */ diff -u -r -N squid-4.0.23/src/store/Disks.h squid-4.0.24/src/store/Disks.h --- squid-4.0.23/src/store/Disks.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Disks.h 2018-03-08 02:16:46.000000000 +1300 @@ -36,10 +36,10 @@ virtual bool dereference(StoreEntry &e) override; virtual void updateHeaders(StoreEntry *) override; virtual void maintain() override; - virtual bool anchorCollapsed(StoreEntry &e, bool &inSync) override; - virtual bool updateCollapsed(StoreEntry &e) override; - virtual void markForUnlink(StoreEntry &) override; - virtual void unlink(StoreEntry &) override; + virtual bool anchorToCache(StoreEntry &e, bool &inSync) override; + virtual bool updateAnchored(StoreEntry &) override; + virtual void evictCached(StoreEntry &) override; + virtual void evictIfFound(const cache_key *) override; virtual int callback() override; /// slowly calculate (and cache) hi/lo watermarks and similar limits @@ -49,6 +49,8 @@ /// reduce the risk of selecting the wrong disk cache for the growing entry. int64_t accumulateMore(const StoreEntry&) const; virtual bool smpAware() const override; + /// whether any of disk caches has entry with e.key + bool hasReadableEntry(const StoreEntry &) const; private: /* migration logic */ diff -u -r -N squid-4.0.23/src/store/forward.h squid-4.0.24/src/store/forward.h --- squid-4.0.23/src/store/forward.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/forward.h 2018-03-08 02:16:46.000000000 +1300 @@ -36,12 +36,16 @@ namespace Store { +/// cache "I/O" direction and status +enum IoStatus { ioUndecided, ioWriting, ioReading, ioDone }; + class Storage; class Controller; class Controlled; class Disks; class Disk; class DiskConfig; +class EntryGuard; typedef ::StoreEntry Entry; typedef ::MemStore Memory; diff -u -r -N squid-4.0.23/src/store/id_rewriters/file/storeid_file_rewrite.8 squid-4.0.24/src/store/id_rewriters/file/storeid_file_rewrite.8 --- squid-4.0.23/src/store/id_rewriters/file/storeid_file_rewrite.8 2018-01-20 02:38:23.000000000 +1300 +++ squid-4.0.24/src/store/id_rewriters/file/storeid_file_rewrite.8 2018-03-08 02:33:25.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "STOREID_FILE_REWRITE 8" -.TH STOREID_FILE_REWRITE 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH STOREID_FILE_REWRITE 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/src/store/Storage.h squid-4.0.24/src/store/Storage.h --- squid-4.0.23/src/store/Storage.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store/Storage.h 2018-03-08 02:16:46.000000000 +1300 @@ -10,7 +10,9 @@ #define SQUID_STORE_STORAGE_H #include "base/RefCount.h" +#include "http/RequestMethod.h" #include "store/forward.h" +#include "store_key_md5.h" class StoreInfoStats; @@ -30,9 +32,6 @@ /// use readable() and writable() methods. virtual void init() = 0; - /// Retrieve a store entry from the store (blocking) - virtual StoreEntry *get(const cache_key *) = 0; - /** * The maximum size the store will support in normal use. Inaccuracy is * permitted, but may throw estimates for memory etc out of whack. @@ -60,11 +59,14 @@ */ virtual void stat(StoreEntry &e) const = 0; - /// expect an unlink() call after the entry becomes idle - virtual void markForUnlink(StoreEntry &e) = 0; - - /// remove the entry from the store - virtual void unlink(StoreEntry &e) = 0; + /// Prevent new get() calls from returning the matching entry. + /// If the matching entry is unused, it may be removed from the store now. + /// The store entry is matched using either `e` attachment info or `e.key`. + virtual void evictCached(StoreEntry &e) = 0; + + /// An evictCached() equivalent for callers that did not get() a StoreEntry. + /// Callers with StoreEntry objects must use evictCached() instead. + virtual void evictIfFound(const cache_key *) = 0; /// called once every main loop iteration; TODO: Move to UFS code. virtual int callback() { return 0; } diff -u -r -N squid-4.0.23/src/store.cc squid-4.0.24/src/store.cc --- squid-4.0.23/src/store.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store.cc 2018-03-08 02:16:46.000000000 +1300 @@ -9,6 +9,7 @@ /* DEBUG: section 20 Storage Manager */ #include "squid.h" +#include "base/TextException.h" #include "CacheDigest.h" #include "CacheManager.h" #include "comm/Connection.h" @@ -138,35 +139,36 @@ pool->freeOne(address); } -void +bool StoreEntry::makePublic(const KeyScope scope) { /* This object can be cached for a long time */ - if (!EBIT_TEST(flags, RELEASE_REQUEST)) - setPublicKey(scope); + return !EBIT_TEST(flags, RELEASE_REQUEST) && setPublicKey(scope); } void StoreEntry::makePrivate(const bool shareable) { - /* This object should never be cached at all */ - expireNow(); releaseRequest(shareable); /* delete object when not used */ } void StoreEntry::clearPrivate() { + assert(!EBIT_TEST(flags, RELEASE_REQUEST)); EBIT_CLR(flags, KEY_PRIVATE); shareableWhenPrivate = false; } -void +bool StoreEntry::cacheNegatively() { /* This object may be negatively cached */ - negativeCache(); - makePublic(); + if (makePublic()) { + negativeCache(); + return true; + } + return false; } size_t @@ -282,7 +284,7 @@ /* the object has completed. */ if (mem_obj->inmem_lo == 0 && !isEmpty()) { - if (swap_status == SWAPOUT_DONE) { + if (swappedOut()) { debugs(20,7, HERE << mem_obj << " lo: " << mem_obj->inmem_lo << " hi: " << mem_obj->endOffset() << " size: " << mem_obj->object_sz); if (mem_obj->endOffset() == mem_obj->object_sz) { /* hot object fully swapped in (XXX: or swapped out?) */ @@ -369,15 +371,15 @@ void StoreEntry::destroyMemObject() { - debugs(20, 3, HERE << "destroyMemObject " << mem_obj); + debugs(20, 3, mem_obj << " in " << *this); - if (MemObject *mem = mem_obj) { - // Store::Root() is FATALly missing during shutdown - if (mem->xitTable.index >= 0 && !shutting_down) - Store::Root().transientsDisconnect(*mem); - if (mem->memCache.index >= 0 && !shutting_down) - Store::Root().memoryDisconnect(*this); + // Store::Root() is FATALly missing during shutdown + if (hasTransients() && !shutting_down) + Store::Root().transientsDisconnect(*this); + if (hasMemStore() && !shutting_down) + Store::Root().memoryDisconnect(*this); + if (MemObject *mem = mem_obj) { setMemStatus(NOT_IN_MEMORY); mem_obj = NULL; delete mem; @@ -395,7 +397,7 @@ return; // Store::Root() is FATALly missing during shutdown - if (e->swap_filen >= 0 && !shutting_down) + if (e->hasDisk() && !shutting_down) e->disk().disconnect(*e); e->destroyMemObject(); @@ -413,6 +415,7 @@ StoreEntry::hashInsert(const cache_key * someKey) { debugs(20, 3, "StoreEntry::hashInsert: Inserting Entry " << *this << " key '" << storeKeyText(someKey) << "'"); + assert(!key); key = storeKeyDup(someKey); hash_join(store_table, this); } @@ -429,21 +432,6 @@ /* -------------------------------------------------------------------------- */ -/* get rid of memory copy of the object */ -void -StoreEntry::purgeMem() -{ - if (mem_obj == NULL) - return; - - debugs(20, 3, "StoreEntry::purgeMem: Freeing memory-copy of " << getMD5Text()); - - Store::Root().memoryUnlink(*this); - - if (swap_status != SWAPOUT_DONE) - release(); -} - void StoreEntry::lock(const char *context) { @@ -458,27 +446,15 @@ } void -StoreEntry::setReleaseFlag() -{ - if (EBIT_TEST(flags, RELEASE_REQUEST)) - return; - - debugs(20, 3, "StoreEntry::setReleaseFlag: '" << getMD5Text() << "'"); - - EBIT_SET(flags, RELEASE_REQUEST); - - Store::Root().markForUnlink(*this); -} - -void StoreEntry::releaseRequest(const bool shareable) { + debugs(20, 3, shareable << ' ' << *this); + if (!shareable) + shareableWhenPrivate = false; // may already be false if (EBIT_TEST(flags, RELEASE_REQUEST)) return; - setReleaseFlag(); // makes validToSend() false, preventing future hits - - setPrivateKey(shareable); + setPrivateKey(shareable, true); } int @@ -492,21 +468,31 @@ if (lock_count) return (int) lock_count; - if (store_status == STORE_PENDING) - setReleaseFlag(); + abandon(context); + return 0; +} +/// keep the unlocked StoreEntry object in the local store_table (if needed) or +/// delete it (otherwise) +void +StoreEntry::doAbandon(const char *context) +{ + debugs(20, 5, *this << " via " << (context ? context : "somebody")); + assert(!locked()); assert(storePendingNClients(this) == 0); - if (EBIT_TEST(flags, RELEASE_REQUEST)) { + // Both aborted local writers and aborted local readers (of remote writers) + // are STORE_PENDING, but aborted readers should never release(). + if (EBIT_TEST(flags, RELEASE_REQUEST) || + (store_status == STORE_PENDING && !Store::Root().transientsReader(*this))) { this->release(); - return 0; + return; } if (EBIT_TEST(flags, KEY_PRIVATE)) debugs(20, DBG_IMPORTANT, "WARNING: " << __FILE__ << ":" << __LINE__ << ": found KEY_PRIVATE"); Store::Root().handleIdleEntry(*this); // may delete us - return 0; } void @@ -548,13 +534,13 @@ StoreEntry * storeGetPublic(const char *uri, const HttpRequestMethod& method) { - return Store::Root().get(storeKeyPublic(uri, method)); + return Store::Root().find(storeKeyPublic(uri, method)); } StoreEntry * storeGetPublicByRequestMethod(HttpRequest * req, const HttpRequestMethod& method, const KeyScope keyScope) { - return Store::Root().get(storeKeyPublicByRequestMethod(req, method, keyScope)); + return Store::Root().find(storeKeyPublicByRequestMethod(req, method, keyScope)); } StoreEntry * @@ -590,22 +576,19 @@ * concept'. */ void -StoreEntry::setPrivateKey(const bool shareable) +StoreEntry::setPrivateKey(const bool shareable, const bool permanent) { - if (key && EBIT_TEST(flags, KEY_PRIVATE)) { - // The entry is already private, but it may be still shareable. - if (!shareable) - shareableWhenPrivate = false; + debugs(20, 3, shareable << permanent << ' ' << *this); + if (permanent) + EBIT_SET(flags, RELEASE_REQUEST); // may already be set + if (!shareable) + shareableWhenPrivate = false; // may already be false + + if (EBIT_TEST(flags, KEY_PRIVATE)) return; - } if (key) { - setReleaseFlag(); // will markForUnlink(); all caches/workers will know - - // TODO: move into SwapDir::markForUnlink() already called by Root() - if (swap_filen > -1) - storeDirSwapLog(this, SWAP_LOG_DEL); - + Store::Root().evictCached(*this); // all caches/workers will know hashDelete(); } @@ -619,11 +602,12 @@ hashInsert(newkey); } -void +bool StoreEntry::setPublicKey(const KeyScope scope) { + debugs(20, 3, *this); if (key && !EBIT_TEST(flags, KEY_PRIVATE)) - return; /* is already public */ + return true; // already public assert(mem_obj); @@ -645,8 +629,17 @@ assert(!EBIT_TEST(flags, RELEASE_REQUEST)); - adjustVary(); - forcePublicKey(calcPublicKey(scope)); + try { + EntryGuard newVaryMarker(adjustVary(), "setPublicKey+failure"); + const cache_key *pubKey = calcPublicKey(scope); + Store::Root().addWriting(this, pubKey); + forcePublicKey(pubKey); + newVaryMarker.unlockAndReset("setPublicKey+success"); + return true; + } catch (const std::exception &ex) { + debugs(20, 2, "for " << *this << " failed: " << ex.what()); + } + return false; } void @@ -669,14 +662,13 @@ void StoreEntry::forcePublicKey(const cache_key *newkey) { + debugs(20, 3, storeKeyText(newkey) << " for " << *this); + assert(mem_obj); + if (StoreEntry *e2 = (StoreEntry *)hash_lookup(store_table, newkey)) { assert(e2 != this); - debugs(20, 3, "Making old " << *e2 << " private."); - - // TODO: check whether there is any sense in keeping old entry - // shareable here. Leaving it non-shareable for now. - e2->setPrivateKey(false); - e2->release(false); + debugs(20, 3, "releasing clashing " << *e2); + e2->release(true); } if (key) @@ -684,9 +676,10 @@ clearPrivate(); + assert(mem_obj->hasUris()); hashInsert(newkey); - if (swap_filen > -1) + if (hasDisk()) storeDirSwapLog(this, SWAP_LOG_ADD); } @@ -703,13 +696,15 @@ /// Updates mem_obj->request->vary_headers to reflect the current Vary. /// The vary_headers field is used to calculate the Vary marker key. /// Releases the old Vary marker with an outdated key (if any). -void +/// \returns new (locked) Vary marker StoreEntry or, if none was needed, nil +/// \throws std::exception on failures +StoreEntry * StoreEntry::adjustVary() { assert(mem_obj); if (!mem_obj->request) - return; + return nullptr; HttpRequest *request = mem_obj->request; @@ -723,7 +718,7 @@ */ request->vary_headers.clear(); /* free old "bad" variance key */ if (StoreEntry *pe = storeGetPublic(mem_obj->storeId(), mem_obj->method)) - pe->release(); + pe->release(true); } /* Make sure the request knows the variance status */ @@ -735,12 +730,19 @@ // We should add/use storeHas() API or lock/unlock those entries. if (!mem_obj->vary_headers.isEmpty() && !storeGetPublic(mem_obj->storeId(), mem_obj->method)) { /* Create "vary" base object */ - String vary; StoreEntry *pe = storeCreateEntry(mem_obj->storeId(), mem_obj->logUri(), request->flags, request->method); + // XXX: storeCreateEntry() already tries to make `pe` public under + // certain conditions. If those conditions do not apply to Vary markers, + // then refactor to call storeCreatePureEntry() above. Otherwise, + // refactor to simply check whether `pe` is already public below. + if (!pe->makePublic()) { + pe->unlock("StoreEntry::adjustVary+failed_makePublic"); + throw TexcHere("failed to make Vary marker public"); + } /* We are allowed to do this typecast */ HttpReply *rep = new HttpReply; rep->setHeaders(Http::scOkay, "Internal marker object", "x-squid-internal/vary", -1, -1, squid_curtime + 100000); - vary = mem_obj->getReply()->header.getList(Http::HdrType::VARY); + String vary = mem_obj->getReply()->header.getList(Http::HdrType::VARY); if (vary.size()) { /* Again, we own this structure layout */ @@ -758,22 +760,21 @@ } #endif - pe->replaceHttpReply(rep, false); // no write until key is public + pe->replaceHttpReply(rep, false); // no write until timestampsSet() pe->timestampsSet(); - pe->makePublic(); - - pe->startWriting(); // after makePublic() + pe->startWriting(); // after timestampsSet() pe->complete(); - pe->unlock("StoreEntry::forcePublicKey+Vary"); + return pe; } + return nullptr; } StoreEntry * -storeCreatePureEntry(const char *url, const char *log_url, const RequestFlags &flags, const HttpRequestMethod& method) +storeCreatePureEntry(const char *url, const char *log_url, const HttpRequestMethod& method) { StoreEntry *e = NULL; debugs(20, 3, "storeCreateEntry: '" << url << "'"); @@ -781,12 +782,6 @@ e = new StoreEntry(); e->createMemObject(url, log_url, method); - if (flags.cachable) { - EBIT_CLR(e->flags, RELEASE_REQUEST); - } else { - e->releaseRequest(); - } - e->store_status = STORE_PENDING; e->refcount = 0; e->lastref = squid_curtime; @@ -799,14 +794,13 @@ StoreEntry * storeCreateEntry(const char *url, const char *logUrl, const RequestFlags &flags, const HttpRequestMethod& method) { - StoreEntry *e = storeCreatePureEntry(url, logUrl, flags, method); + StoreEntry *e = storeCreatePureEntry(url, logUrl, method); e->lock("storeCreateEntry"); - if (neighbors_do_private_keys || !flags.hierarchical) - e->setPrivateKey(false); - else - e->setPublicKey(); + if (!neighbors_do_private_keys && flags.hierarchical && flags.cachable && e->setPublicKey()) + return e; + e->setPrivateKey(false, !flags.cachable); return e; } @@ -865,26 +859,18 @@ *buf = 0; int x; -#ifdef VA_COPY - va_args ap; + va_list ap; /* Fix of bug 753r. The value of vargs is undefined * after vsnprintf() returns. Make a copy of vargs * incase we loop around and call vsnprintf() again. */ - VA_COPY(ap,vargs); + va_copy(ap,vargs); errno = 0; if ((x = vsnprintf(buf, sizeof(buf), fmt, ap)) < 0) { fatal(xstrerr(errno)); return; } va_end(ap); -#else /* VA_COPY */ - errno = 0; - if ((x = vsnprintf(buf, sizeof(buf), fmt, vargs)) < 0) { - fatal(xstrerr(errno)); - return; - } -#endif /*VA_COPY*/ if (x < static_cast(sizeof(buf))) { append(buf, x); @@ -1021,11 +1007,10 @@ } else if (EBIT_TEST(flags, KEY_PRIVATE)) { debugs(20, 3, "StoreEntry::checkCachable: NO: private key"); ++store_check_cachable_hist.no.private_key; - } else if (swap_status != SWAPOUT_NONE) { + } else if (hasDisk()) { /* - * here we checked the swap_status because the remaining - * cases are only relevant only if we haven't started swapping - * out the object yet. + * the remaining cases are only relevant if we haven't + * started swapping out the object yet. */ return 1; } else if (storeTooManyDiskFilesOpen()) { @@ -1185,44 +1170,8 @@ storeGetMemSpace(int size) { PROF_start(storeGetMemSpace); - StoreEntry *e = NULL; - int released = 0; - static time_t last_check = 0; - size_t pages_needed; - RemovalPurgeWalker *walker; - - if (squid_curtime == last_check) { - PROF_stop(storeGetMemSpace); - return; - } - - last_check = squid_curtime; - - pages_needed = (size + SM_PAGE_SIZE-1) / SM_PAGE_SIZE; - - if (mem_node::InUseCount() + pages_needed < store_pages_max) { - PROF_stop(storeGetMemSpace); - return; - } - - debugs(20, 2, "storeGetMemSpace: Starting, need " << pages_needed << - " pages"); - - /* XXX what to set as max_scan here? */ - walker = mem_policy->PurgeInit(mem_policy, 100000); - - while ((e = walker->Next(walker))) { - e->purgeMem(); - ++released; - - if (mem_node::InUseCount() + pages_needed < store_pages_max) - break; - } - - walker->Done(walker); - debugs(20, 3, "storeGetMemSpace stats:"); - debugs(20, 3, " " << std::setw(6) << hot_obj_count << " HOT objects"); - debugs(20, 3, " " << std::setw(6) << released << " were released"); + if (!shutting_down) // Store::Root() is FATALly missing during shutdown + Store::Root().freeMemorySpace(size); PROF_stop(storeGetMemSpace); } @@ -1245,44 +1194,33 @@ #define MAINTAIN_MAX_SCAN 1024 #define MAINTAIN_MAX_REMOVE 64 -/* release an object from a cache */ void StoreEntry::release(const bool shareable) { PROF_start(storeRelease); - debugs(20, 3, "releasing " << *this << ' ' << getMD5Text()); + debugs(20, 3, shareable << ' ' << *this << ' ' << getMD5Text()); /* If, for any reason we can't discard this object because of an * outstanding request, mark it for pending release */ if (locked()) { - expireNow(); - debugs(20, 3, "storeRelease: Only setting RELEASE_REQUEST bit"); releaseRequest(shareable); PROF_stop(storeRelease); return; } - if (Store::Controller::store_dirs_rebuilding && swap_filen > -1) { + if (Store::Controller::store_dirs_rebuilding && hasDisk()) { /* TODO: Teach disk stores to handle releases during rebuild instead. */ - Store::Root().memoryUnlink(*this); - - setPrivateKey(shareable); - // lock the entry until rebuilding is done lock("storeLateRelease"); - setReleaseFlag(); + releaseRequest(shareable); LateReleaseStack.push(this); + PROF_stop(storeRelease); return; } storeLog(STORE_LOG_RELEASE, this); - if (swap_filen > -1 && !EBIT_TEST(flags, KEY_PRIVATE)) { - // log before unlink() below clears swap_filen - storeDirSwapLog(this, SWAP_LOG_DEL); - } - - Store::Root().unlink(*this); + Store::Root().evictCached(*this); destroyStoreEntry(static_cast(this)); PROF_stop(storeRelease); } @@ -1421,7 +1359,7 @@ if (mem_obj->inmem_lo != 0) return 0; - if (!Config.onoff.memory_cache_first && swap_status == SWAPOUT_DONE && refcount == 1) + if (!Config.onoff.memory_cache_first && swappedOut() && refcount == 1) return 0; return 1; @@ -1495,7 +1433,7 @@ return 0; // now check that the entry has a cache backing or is collapsed - if (swap_filen > -1) // backed by a disk cache + if (hasDisk()) // backed by a disk cache return 1; if (swappingOut()) // will be backed by a disk cache @@ -1846,7 +1784,7 @@ flush(); complete(); negativeCache(); - releaseRequest(); + releaseRequest(false); // if it is safe to negatively cache, sharing is OK unlock("StoreEntry::storeErrorResponse"); } @@ -1913,13 +1851,13 @@ void StoreEntry::transientsAbandonmentCheck() { - if (mem_obj && !mem_obj->smpCollapsed && // this worker is responsible - mem_obj->xitTable.index >= 0 && // other workers may be interested - mem_obj->memCache.index < 0 && // rejected by the shared memory cache + if (mem_obj && !Store::Root().transientsReader(*this) && // this worker is responsible + hasTransients() && // other workers may be interested + !hasMemStore() && // rejected by the shared memory cache mem_obj->swapout.decision == MemObject::SwapOut::swImpossible) { debugs(20, 7, "cannot be shared: " << *this); if (!shutting_down) // Store::Root() is FATALly missing during shutdown - Store::Root().transientsAbandon(*this); + Store::Root().stopSharing(*this); } } @@ -2056,12 +1994,58 @@ Store::Disk & StoreEntry::disk() const { - assert(0 <= swap_dirn && swap_dirn < Config.cacheSwap.n_configured); + assert(hasDisk()); const RefCount &sd = INDEXSD(swap_dirn); assert(sd); return *sd; } +bool +StoreEntry::hasDisk(const sdirno dirn, const sfileno filen) const +{ + checkDisk(); + if (dirn < 0 && filen < 0) + return swap_dirn >= 0; + Must(dirn >= 0); + const bool matchingDisk = (swap_dirn == dirn); + return filen < 0 ? matchingDisk : (matchingDisk && swap_filen == filen); +} + +void +StoreEntry::attachToDisk(const sdirno dirn, const sfileno fno, const swap_status_t status) +{ + debugs(88, 3, "attaching entry with key " << getMD5Text() << " : " << + swapStatusStr[status] << " " << dirn << " " << + std::hex << std::setw(8) << std::setfill('0') << + std::uppercase << fno); + checkDisk(); + swap_dirn = dirn; + swap_filen = fno; + swap_status = status; + checkDisk(); +} + +void +StoreEntry::detachFromDisk() +{ + swap_dirn = -1; + swap_filen = -1; + swap_status = SWAPOUT_NONE; +} + +void +StoreEntry::checkDisk() const +{ + const bool ok = (swap_dirn < 0) == (swap_filen < 0) && + (swap_dirn < 0) == (swap_status == SWAPOUT_NONE) && + (swap_dirn < 0 || swap_dirn < Config.cacheSwap.n_configured); + + if (!ok) { + debugs(88, DBG_IMPORTANT, "ERROR: inconsistent disk entry state " << *this); + throw std::runtime_error("inconsistent disk entry state "); + } +} + /* * return true if the entry is in a state where * it can accept more data (ie with write() method) @@ -2090,16 +2074,41 @@ return buf; } +static std::ostream & +operator <<(std::ostream &os, const Store::IoStatus &io) +{ + switch (io) { + case Store::ioUndecided: + os << 'u'; + break; + case Store::ioReading: + os << 'r'; + break; + case Store::ioWriting: + os << 'w'; + break; + case Store::ioDone: + os << 'o'; + break; + } + return os; +} + std::ostream &operator <<(std::ostream &os, const StoreEntry &e) { os << "e:"; - if (e.mem_obj) { - if (e.mem_obj->xitTable.index > -1) - os << 't' << e.mem_obj->xitTable.index; - if (e.mem_obj->memCache.index > -1) - os << 'm' << e.mem_obj->memCache.index; + if (e.hasTransients()) { + const auto &xitTable = e.mem_obj->xitTable; + os << 't' << xitTable.io << xitTable.index; + } + + if (e.hasMemStore()) { + const auto &memCache = e.mem_obj->memCache; + os << 'm' << memCache.io << memCache.index << '@' << memCache.offset; } + + // Do not use e.hasDisk() here because its checkDisk() call may calls us. if (e.swap_filen > -1 || e.swap_dirn > -1) os << 'd' << e.swap_filen << '@' << e.swap_dirn; @@ -2129,7 +2138,6 @@ if (e.shareableWhenPrivate) os << 'H'; } - if (EBIT_TEST(e.flags, KEY_PRIVATE)) os << 'I'; if (EBIT_TEST(e.flags, ENTRY_FWD_HDR_WAIT)) os << 'W'; if (EBIT_TEST(e.flags, ENTRY_NEGCACHED)) os << 'N'; if (EBIT_TEST(e.flags, ENTRY_VALIDATED)) os << 'V'; @@ -2137,9 +2145,6 @@ if (EBIT_TEST(e.flags, ENTRY_ABORTED)) os << 'A'; } - if (e.mem_obj && e.mem_obj->smpCollapsed) - os << 'O'; - return os << '/' << &e << '*' << e.locks(); } @@ -2171,3 +2176,12 @@ return NULL; } +void +Store::EntryGuard::onException() noexcept +{ + SWALLOW_EXCEPTIONS({ + entry_->releaseRequest(false); + entry_->unlock(context_); + }); +} + diff -u -r -N squid-4.0.23/src/store_client.cc squid-4.0.24/src/store_client.cc --- squid-4.0.23/src/store_client.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_client.cc 2018-03-08 02:16:46.000000000 +1300 @@ -162,7 +162,7 @@ if (getType() == STORE_DISK_CLIENT) { /* assert we'll be able to get the data we want */ /* maybe we should open swapin_sio here */ - assert(entry->swap_filen > -1 || entry->swappingOut()); + assert(entry->hasDisk() || entry->swappingOut()); } } @@ -251,7 +251,7 @@ const int64_t len = entry->objectLen(); // If we do not know the entry length, then we have to open the swap file. - const bool canSwapIn = entry->swap_filen >= 0; + const bool canSwapIn = entry->hasDisk(); if (len < 0) return canSwapIn; @@ -441,7 +441,7 @@ flags.disk_io_pending = true; if (mem->swap_hdr_sz != 0) - if (entry->swap_status == SWAPOUT_WRITING) + if (entry->swappingOut()) assert(mem->swapout.sio->offset() > copyInto.offset + (int64_t)mem->swap_hdr_sz); storeRead(swapin_sio, @@ -667,7 +667,7 @@ dlinkDelete(&sc->node, &mem->clients); -- mem->nclients; - if (e->store_status == STORE_OK && e->swap_status != SWAPOUT_DONE) + if (e->store_status == STORE_OK && !e->swappedOut()) e->swapOut(); if (sc->swapin_sio != NULL) { diff -u -r -N squid-4.0.23/src/Store.h squid-4.0.24/src/Store.h --- squid-4.0.23/src/Store.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/Store.h 2018-03-08 02:16:46.000000000 +1300 @@ -82,28 +82,43 @@ void swapOutDecision(const MemObject::SwapOut::Decision &decision); void abort(); - void makePublic(const KeyScope keyScope = ksDefault); + bool makePublic(const KeyScope keyScope = ksDefault); void makePrivate(const bool shareable); /// A low-level method just resetting "private key" flags. /// To avoid key inconsistency please use forcePublicKey() /// or similar instead. void clearPrivate(); - void setPublicKey(const KeyScope keyScope = ksDefault); + bool setPublicKey(const KeyScope keyScope = ksDefault); /// Resets existing public key to a public key with default scope, /// releasing the old default-scope entry (if any). /// Does nothing if the existing public key already has default scope. void clearPublicKeyScope(); - void setPrivateKey(const bool shareable); + + /// \returns public key (if the entry has it) or nil (otherwise) + const cache_key *publicKey() const { + return (!EBIT_TEST(flags, KEY_PRIVATE)) ? + reinterpret_cast(key): // may be nil + nullptr; + } + + /// Either fills this entry with private key or changes the existing key + /// from public to private. + /// \param permanent whether this entry should be private forever. + void setPrivateKey(const bool shareable, const bool permanent); + void expireNow(); + /// Makes the StoreEntry private and marks the corresponding entry + /// for eventual removal from the Store. void releaseRequest(const bool shareable = false); void negativeCache(); - void cacheNegatively(); /** \todo argh, why both? */ + bool cacheNegatively(); /** \todo argh, why both? */ void invokeHandlers(); - void purgeMem(); void cacheInMemory(); ///< start or continue storing in memory cache void swapOut(); /// whether we are in the process of writing this entry to disk bool swappingOut() const { return swap_status == SWAPOUT_WRITING; } + /// whether the entire entry is now on disk (possibly marked for deletion) + bool swappedOut() const { return swap_status == SWAPOUT_DONE; } void swapOutFileClose(int how); const char *url() const; /// Satisfies cachability requirements shared among disk and RAM caches. @@ -158,6 +173,20 @@ /// the disk this entry is [being] cached on; asserts for entries w/o a disk Store::Disk &disk() const; + /// whether one of this StoreEntry owners has locked the corresponding + /// disk entry (at the specified disk entry coordinates, if any) + bool hasDisk(const sdirno dirn = -1, const sfileno filen = -1) const; + /// Makes hasDisk(dirn, filn) true. The caller should have locked + /// the corresponding disk store entry for reading or writing. + void attachToDisk(const sdirno, const sfileno, const swap_status_t); + /// Makes hasDisk() false. The caller should have unlocked + /// the corresponding disk store entry. + void detachFromDisk(); + + /// whether there is a corresponding locked transients table entry + bool hasTransients() const { return mem_obj && mem_obj->xitTable.index >= 0; } + /// whether there is a corresponding locked shared memory table entry + bool hasMemStore() const { return mem_obj && mem_obj->memCache.index >= 0; } MemObject *mem_obj; RemovalPolicyNode repl; @@ -198,7 +227,6 @@ void *operator new(size_t byteCount); void operator delete(void *address); - void setReleaseFlag(); #if USE_SQUID_ESI ESIElement::Pointer cachedESITree; @@ -220,8 +248,19 @@ /// update last reference timestamp and related Store metadata void touch(); + /// One of the three methods to get rid of an unlocked StoreEntry object. + /// Removes all unlocked (and marks for eventual removal all locked) Store + /// entries, including attached and unattached entries that have our key. + /// Also destroys us if we are unlocked or makes us private otherwise. + /// TODO: remove virtual. virtual void release(const bool shareable = false); + /// One of the three methods to get rid of an unlocked StoreEntry object. + /// May destroy this object if it is unlocked; does nothing otherwise. + /// Unlike release(), may not trigger eviction of underlying store entries, + /// but, unlike destroyStoreEntry(), does honor an earlier release request. + void abandon(const char *context) { if (!locked()) doAbandon(context); } + /// May the caller commit to treating this [previously locked] /// entry as a cache hit? bool mayStartHitting() const { @@ -242,12 +281,17 @@ virtual void flush(); protected: + typedef Store::EntryGuard EntryGuard; + void transientsAbandonmentCheck(); + /// does nothing except throwing if disk-associated data members are inconsistent + void checkDisk() const; private: + void doAbandon(const char *context); bool checkTooBig() const; void forcePublicKey(const cache_key *newkey); - void adjustVary(); + StoreEntry *adjustVary(); const cache_key *calcPublicKey(const KeyScope keyScope); static MemAllocator *pool; @@ -310,9 +354,53 @@ typedef void (*STOREGETCLIENT) (StoreEntry *, void *cbdata); namespace Store { + +/// a smart pointer similar to std::unique_ptr<> that automatically +/// release()s and unlock()s the guarded Entry on stack-unwinding failures +class EntryGuard { +public: + /// \param entry either nil or a locked Entry to manage + /// \param context default unlock() message + EntryGuard(Entry *entry, const char *context): + entry_(entry), context_(context) { + assert(!entry_ || entry_->locked()); + } + + ~EntryGuard() { + if (entry_) { + // something went wrong -- the caller did not unlockAndReset() us + onException(); + } + } + + EntryGuard(EntryGuard &&) = delete; // no copying or moving (for now) + + /// like std::unique_ptr::get() + /// \returns nil or the guarded (locked) entry + Entry *get() { + return entry_; + } + + /// like std::unique_ptr::reset() + /// stops guarding the entry + /// unlocks the entry (which may destroy it) + void unlockAndReset(const char *resetContext = nullptr) { + if (entry_) { + entry_->unlock(resetContext ? resetContext : context_); + entry_ = nullptr; + } + } + +private: + void onException() noexcept; + + Entry *entry_; ///< the guarded Entry or nil + const char *context_; ///< default unlock() message +}; + void Stats(StoreEntry *output); void Maintain(void *unused); -}; +}; // namespace Store /// \ingroup StoreAPI size_t storeEntryInUse(); @@ -338,7 +426,7 @@ /// \ingroup StoreAPI /// Creates a new StoreEntry with mem_obj and sets initial flags/states. -StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&); +StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&); /// \ingroup StoreAPI void storeInit(void); @@ -376,7 +464,10 @@ /// \ingroup StoreAPI void storeReplAdd(const char *, REMOVALPOLICYCREATE *); -/// \ingroup StoreAPI +/// One of the three methods to get rid of an unlocked StoreEntry object. +/// This low-level method ignores lock()ing and release() promises. It never +/// leaves the entry in the local store_table. +/// TODO: Hide by moving its functionality into the StoreEntry destructor. extern FREE destroyStoreEntry; /// \ingroup StoreAPI diff -u -r -N squid-4.0.23/src/store_log.cc squid-4.0.24/src/store_log.cc --- squid-4.0.23/src/store_log.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_log.cc 2018-03-08 02:16:46.000000000 +1300 @@ -57,6 +57,9 @@ String ctype=(reply->content_type.size() ? reply->content_type.termedBuf() : str_unknown); + // mem_obj may still lack logging details; especially in RELEASE cases + const char *logUri = mem->hasUris() ? mem->logUri() : "?"; + logfileLineStart(storelog); logfilePrintf(storelog, "%9d.%03d %-7s %02d %08X %s %4d %9d %9d %9d " SQUIDSTRINGPH " %" PRId64 "/%" PRId64 " " SQUIDSBUFPH " %s\n", (int) current_time.tv_sec, @@ -73,7 +76,7 @@ reply->content_length, e->contentLen(), SQUIDSBUFPRINT(mem->method.image()), - mem->logUri()); + logUri); logfileLineEnd(storelog); } else { /* no mem object. Most RELEASE cases */ diff -u -r -N squid-4.0.23/src/store_rebuild.cc squid-4.0.24/src/store_rebuild.cc --- squid-4.0.23/src/store_rebuild.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_rebuild.cc 2018-03-08 02:16:46.000000000 +1300 @@ -75,7 +75,7 @@ * Calling StoreEntry->release() has no effect because we're * still in 'store_rebuilding' state */ - if (e->swap_filen < 0) + if (!e->hasDisk()) continue; if (opt_store_doublecheck) @@ -357,50 +357,5 @@ } return true; -} - -bool -storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key, StoreRebuildData &stats) -{ - /* this needs to become - * 1) unpack url - * 2) make synthetic request with headers ?? or otherwise search - * for a matching object in the store - * TODO FIXME change to new async api - * TODO FIXME I think there is a race condition here with the - * async api : - * store A reads in object foo, searchs for it, and finds nothing. - * store B reads in object foo, searchs for it, finds nothing. - * store A gets called back with nothing, so registers the object - * store B gets called back with nothing, so registers the object, - * which will conflict when the in core index gets around to scanning - * store B. - * - * this suggests that rather than searching for duplicates, the - * index rebuild should just assume its the most recent accurate - * store entry and whoever indexes the stores handles duplicates. - */ - if (StoreEntry *e = Store::Root().get(key)) { - - if (e->lastref >= tmpe.lastref) { - /* key already exists, old entry is newer */ - /* keep old, ignore new */ - ++stats.dupcount; - - // For some stores, get() creates/unpacks a store entry. Signal - // such stores that we will no longer use the get() result: - e->lock("storeRebuildKeepEntry"); - e->unlock("storeRebuildKeepEntry"); - - return false; - } else { - /* URL already exists, this swapfile not being used */ - /* junk old, load new */ - e->release(); /* release old entry */ - ++stats.dupcount; - } - } - - return true; } diff -u -r -N squid-4.0.23/src/store_rebuild.h squid-4.0.24/src/store_rebuild.h --- squid-4.0.23/src/store_rebuild.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_rebuild.h 2018-03-08 02:16:46.000000000 +1300 @@ -42,8 +42,6 @@ bool storeRebuildLoadEntry(int fd, int diskIndex, MemBuf &buf, StoreRebuildData &counts); /// parses entry buffer and validates entry metadata; fills e on success bool storeRebuildParseEntry(MemBuf &buf, StoreEntry &e, cache_key *key, StoreRebuildData &counts, uint64_t expectedSize); -/// checks whether the loaded entry should be kept; updates counters -bool storeRebuildKeepEntry(const StoreEntry &e, const cache_key *key, StoreRebuildData &counts); #endif /* SQUID_STORE_REBUILD_H_ */ diff -u -r -N squid-4.0.23/src/store_swapin.cc squid-4.0.24/src/store_swapin.cc --- squid-4.0.23/src/store_swapin.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_swapin.cc 2018-03-08 02:16:46.000000000 +1300 @@ -31,22 +31,14 @@ if (e->mem_status != NOT_IN_MEMORY) debugs(20, 3, HERE << "already IN_MEMORY"); - debugs(20, 3, "storeSwapInStart: called for : " << e->swap_dirn << " " << - std::hex << std::setw(8) << std::setfill('0') << std::uppercase << - e->swap_filen << " " << e->getMD5Text()); + debugs(20, 3, *e << " " << e->getMD5Text()); - if (e->swap_status != SWAPOUT_WRITING && e->swap_status != SWAPOUT_DONE) { - debugs(20, DBG_IMPORTANT, "storeSwapInStart: bad swap_status (" << swapStatusStr[e->swap_status] << ")"); - return; - } - - if (e->swap_filen < 0) { - debugs(20, DBG_IMPORTANT, "storeSwapInStart: swap_filen < 0"); + if (!e->hasDisk()) { + debugs(20, DBG_IMPORTANT, "BUG: Attempt to swap in a not-stored entry " << *e << ". Salvaged."); return; } assert(e->mem_obj != NULL); - debugs(20, 3, "storeSwapInStart: Opening fileno " << std::hex << std::setw(8) << std::setfill('0') << std::uppercase << e->swap_filen); sc->swapin_sio = storeOpen(e, storeSwapInFileNotify, storeSwapInFileClosed, sc); } diff -u -r -N squid-4.0.23/src/store_swapout.cc squid-4.0.24/src/store_swapout.cc --- squid-4.0.23/src/store_swapout.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/store_swapout.cc 2018-03-08 02:16:46.000000000 +1300 @@ -10,6 +10,7 @@ #include "squid.h" #include "cbdata.h" +#include "CollapsedForwarding.h" #include "globals.h" #include "Store.h" #include "StoreClient.h" @@ -46,7 +47,6 @@ debugs(20, 5, "storeSwapOutStart: Begin SwapOut '" << e->url() << "' to dirno " << e->swap_dirn << ", fileno " << std::hex << std::setw(8) << std::setfill('0') << std::uppercase << e->swap_filen); - e->swap_status = SWAPOUT_WRITING; e->swapOutDecision(MemObject::SwapOut::swStarted); /* If we start swapping out objects with OutOfBand Metadata, * then this code needs changing @@ -65,6 +65,7 @@ sio = storeCreate(e, storeSwapOutFileNotify, storeSwapOutFileClosed, c); if (sio == NULL) { + assert(!e->hasDisk()); e->swap_status = SWAPOUT_NONE; e->swapOutDecision(MemObject::SwapOut::swImpossible); delete c; @@ -79,14 +80,13 @@ e->lock("storeSwapOutStart"); /* Pick up the file number if it was assigned immediately */ - e->swap_filen = mem->swapout.sio->swap_filen; - - e->swap_dirn = mem->swapout.sio->swap_dirn; + e->attachToDisk(mem->swapout.sio->swap_dirn, mem->swapout.sio->swap_filen, SWAPOUT_WRITING); /* write out the swap metadata */ storeIOWrite(mem->swapout.sio, buf, mem->swap_hdr_sz, 0, xfree_cppwrapper); } +/// XXX: unused, see a related StoreIOState::file_callback static void storeSwapOutFileNotify(void *data, int errflag, StoreIOState::Pointer self) { @@ -94,11 +94,11 @@ static_cast(data)->unwrap(&e); MemObject *mem = e->mem_obj; - assert(e->swap_status == SWAPOUT_WRITING); + assert(e->swappingOut()); assert(mem); assert(mem->swapout.sio == self); assert(errflag == 0); - assert(e->swap_filen < 0); // if this fails, call SwapDir::disconnect(e) + assert(!e->hasDisk()); // if this fails, call SwapDir::disconnect(e) e->swap_filen = mem->swapout.sio->swap_filen; e->swap_dirn = mem->swapout.sio->swap_dirn; } @@ -146,7 +146,7 @@ -1, memNodeWriteComplete); - if (!ok || anEntry->swap_status != SWAPOUT_WRITING) + if (!ok || !anEntry->swappingOut()) return false; int64_t swapout_size = mem->endOffset() - mem->swapout.queue_offset; @@ -210,7 +210,7 @@ } #endif - if (swap_status == SWAPOUT_WRITING) + if (swappingOut()) assert(mem_obj->inmem_lo <= mem_obj->objectBytesOnDisk() ); // buffered bytes we have not swapped out yet @@ -242,7 +242,7 @@ } /* Ok, we have stuff to swap out. Is there a swapout.sio open? */ - if (swap_status == SWAPOUT_NONE) { + if (!hasDisk()) { assert(mem_obj->swapout.sio == NULL); assert(mem_obj->inmem_lo == 0); storeSwapOutStart(this); // sets SwapOut::swImpossible on failures @@ -288,7 +288,7 @@ MemObject *mem = e->mem_obj; assert(mem->swapout.sio == self); - assert(e->swap_status == SWAPOUT_WRITING); + assert(e->swappingOut()); // if object_size is still unknown, the entry was probably aborted if (errflag || e->objectLen() < 0) { @@ -304,12 +304,8 @@ storeConfigure(); } - if (e->swap_filen >= 0) - e->disk().unlink(*e); - - assert(e->swap_status == SWAPOUT_NONE); - - e->releaseRequest(); + e->disk().finalizeSwapoutFailure(*e); + e->releaseRequest(); // TODO: Keep the memory entry (if any) } else { /* swapping complete */ debugs(20, 3, "storeSwapOutFileClosed: SwapOut complete: '" << e->url() << "' to " << @@ -320,7 +316,7 @@ e->swap_file_sz = e->objectLen() + mem->swap_hdr_sz; e->swap_status = SWAPOUT_DONE; - e->disk().swappedOut(*e); + e->disk().finalizeSwapoutSuccess(*e); // XXX: For some Stores, it is pointless to re-check cachability here // and it leads to double counts in store_check_cachable_hist. We need @@ -334,6 +330,7 @@ ++statCounter.swap.outs; } + Store::Root().transientsCompleteWriting(*e); debugs(20, 3, "storeSwapOutFileClosed: " << __FILE__ << ":" << __LINE__); mem->swapout.sio = NULL; e->unlock("storeSwapOutFileClosed"); @@ -358,19 +355,26 @@ return false; } - // if we swapped out already, do not start over - if (swap_status == SWAPOUT_DONE) { + // if we are swapping out or swapped out already, do not start over + if (hasDisk() || Store::Root().hasReadableDiskEntry(*this)) { debugs(20, 3, "already did"); swapOutDecision(MemObject::SwapOut::swImpossible); return false; } - // if we stared swapping out already, do not start over + // if we have just stared swapping out (attachToDisk() has not been + // called), do not start over if (decision == MemObject::SwapOut::swStarted) { debugs(20, 3, "already started"); swapOutDecision(MemObject::SwapOut::swImpossible); return false; } + + if (Store::Root().markedForDeletionAndAbandoned(*this)) { + debugs(20, 3, "marked for deletion and abandoned"); + swapOutDecision(MemObject::SwapOut::swImpossible); + return false; + } // if we decided that swapout is possible, do not repeat same checks if (decision == MemObject::SwapOut::swPossible) { diff -u -r -N squid-4.0.23/src/tests/stub_CollapsedForwarding.cc squid-4.0.24/src/tests/stub_CollapsedForwarding.cc --- squid-4.0.23/src/tests/stub_CollapsedForwarding.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_CollapsedForwarding.cc 2018-03-08 02:16:46.000000000 +1300 @@ -12,5 +12,6 @@ #define STUB_API "CollapsedForwarding.cc" #include "tests/STUB.h" -void CollapsedForwarding::Broadcast(StoreEntry const&) STUB +void CollapsedForwarding::Broadcast(StoreEntry const&, const bool) STUB +void CollapsedForwarding::Broadcast(const sfileno, const bool) STUB diff -u -r -N squid-4.0.23/src/tests/stub_libsecurity.cc squid-4.0.24/src/tests/stub_libsecurity.cc --- squid-4.0.23/src/tests/stub_libsecurity.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_libsecurity.cc 2018-03-08 02:16:46.000000000 +1300 @@ -31,6 +31,12 @@ Security::HandshakeParser::HandshakeParser() STUB bool Security::HandshakeParser::parseHello(const SBuf &) STUB_RETVAL(false) +#include "security/KeyData.h" +namespace Security +{ +void KeyData::loadFromFiles(const AnyP::PortCfg &, const char *) STUB +} + #include "security/NegotiationHistory.h" Security::NegotiationHistory::NegotiationHistory() STUB void Security::NegotiationHistory::retrieveNegotiatedInfo(const Security::SessionPointer &) STUB @@ -91,8 +97,9 @@ void Security::ServerOptions::parse(const char *) STUB void Security::ServerOptions::dumpCfg(Packable *, const char *) const STUB Security::ContextPointer Security::ServerOptions::createBlankContext() const STUB_RETVAL(Security::ContextPointer()) +void Security::ServerOptions::initServerContexts(AnyP::PortCfg&) STUB bool Security::ServerOptions::createStaticServerContext(AnyP::PortCfg &) STUB_RETVAL(false) -void Security::ServerOptions::createSigningContexts(AnyP::PortCfg &) STUB +void Security::ServerOptions::createSigningContexts(const AnyP::PortCfg &) STUB bool Security::ServerOptions::updateContextConfig(Security::ContextPointer &) STUB_RETVAL(false) void Security::ServerOptions::updateContextEecdh(Security::ContextPointer &) STUB void Security::ServerOptions::updateContextClientCa(Security::ContextPointer &) STUB diff -u -r -N squid-4.0.23/src/tests/stub_libsslsquid.cc squid-4.0.24/src/tests/stub_libsslsquid.cc --- squid-4.0.23/src/tests/stub_libsslsquid.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_libsslsquid.cc 2018-03-08 02:16:46.000000000 +1300 @@ -50,6 +50,7 @@ #include "ssl/support.h" namespace Ssl { +int AskPasswordCb(char *, int, int, void *) STUB_RETVAL(0) bool InitServerContext(Security::ContextPointer &, AnyP::PortCfg &) STUB_RETVAL(false) bool InitClientContext(Security::ContextPointer &, Security::PeerOptions &, const char *) STUB_RETVAL(false) void SetupVerifyCallback(Security::ContextPointer &) STUB @@ -70,8 +71,6 @@ Security::ContextPointer GenerateSslContext(CertificateProperties const &, Security::ServerOptions &, bool) STUB_RETVAL(Security::ContextPointer()) bool verifySslCertificate(Security::ContextPointer &, CertificateProperties const &) STUB_RETVAL(false) Security::ContextPointer GenerateSslContextUsingPkeyAndCertFromMemory(const char *, Security::ServerOptions &, bool) STUB_RETVAL(Security::ContextPointer()) -void addChainToSslContext(Security::ContextPointer &, STACK_OF(X509) *) STUB -void readCertChainAndPrivateKeyFromFiles(Security::CertPointer &, Security::PrivateKeyPointer &, Security::CertList &, char const *, char const *) STUB int matchX509CommonNames(X509 *peer_cert, void *check_data, int (*check_func)(void *check_data, ASN1_STRING *cn_data)) STUB_RETVAL(0) bool checkX509ServerValidity(X509 *cert, const char *server) STUB_RETVAL(false) int asn1timeToString(ASN1_TIME *tm, char *buf, int len) STUB_RETVAL(0) diff -u -r -N squid-4.0.23/src/tests/stub_MemStore.cc squid-4.0.24/src/tests/stub_MemStore.cc --- squid-4.0.23/src/tests/stub_MemStore.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_MemStore.cc 2018-03-08 02:16:46.000000000 +1300 @@ -19,7 +19,6 @@ bool MemStore::keepInLocalMemory(const StoreEntry &) const STUB_RETVAL(false) void MemStore::write(StoreEntry &e) STUB void MemStore::completeWriting(StoreEntry &e) STUB -void MemStore::unlink(StoreEntry &e) STUB void MemStore::disconnect(StoreEntry &e) STUB void MemStore::reference(StoreEntry &) STUB void MemStore::updateHeaders(StoreEntry *) STUB @@ -35,7 +34,8 @@ uint64_t MemStore::currentCount() const STUB_RETVAL(0) int64_t MemStore::maxObjectSize() const STUB_RETVAL(0) bool MemStore::dereference(StoreEntry &) STUB_RETVAL(false) -void MemStore::markForUnlink(StoreEntry&) STUB -bool MemStore::anchorCollapsed(StoreEntry&, bool&) STUB_RETVAL(false) -bool MemStore::updateCollapsed(StoreEntry&) STUB_RETVAL(false) +void MemStore::evictCached(StoreEntry&) STUB +void MemStore::evictIfFound(const cache_key *) STUB +bool MemStore::anchorToCache(StoreEntry&, bool&) STUB_RETVAL(false) +bool MemStore::updateAnchored(StoreEntry&) STUB_RETVAL(false) diff -u -r -N squid-4.0.23/src/tests/stub_store.cc squid-4.0.24/src/tests/stub_store.cc --- squid-4.0.23/src/tests/stub_store.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_store.cc 2018-03-08 02:16:46.000000000 +1300 @@ -37,15 +37,14 @@ bool StoreEntry::mayStartSwapOut() STUB_RETVAL(false) void StoreEntry::trimMemory(const bool preserveSwappable) STUB void StoreEntry::abort() STUB -void StoreEntry::makePublic(const KeyScope scope) STUB +bool StoreEntry::makePublic(const KeyScope scope) STUB void StoreEntry::makePrivate(const bool shareable) STUB -void StoreEntry::setPublicKey(const KeyScope scope) STUB -void StoreEntry::setPrivateKey(const bool shareable) STUB +bool StoreEntry::setPublicKey(const KeyScope scope) STUB +void StoreEntry::setPrivateKey(const bool, const bool) STUB void StoreEntry::expireNow() STUB void StoreEntry::releaseRequest(const bool shareable) STUB void StoreEntry::negativeCache() STUB -void StoreEntry::cacheNegatively() STUB -void StoreEntry::purgeMem() STUB +bool StoreEntry::cacheNegatively() STUB void StoreEntry::swapOut() STUB void StoreEntry::swapOutFileClose(int how) STUB const char *StoreEntry::url() const STUB_RETVAL(NULL) @@ -83,7 +82,6 @@ return new StoreEntry(); } void StoreEntry::operator delete(void *address) STUB -void StoreEntry::setReleaseFlag() STUB //#if USE_SQUID_ESI //ESIElement::Pointer StoreEntry::cachedESITree STUB_RETVAL(NULL) //#endif @@ -112,6 +110,8 @@ int Store::Controller::store_dirs_rebuilding = 0; StoreSearch *Store::Controller::search() STUB_RETVAL(NULL) void Store::Controller::maintain() STUB +bool Store::Controller::markedForDeletion(const cache_key *) const STUB_RETVAL(false) +void Store::Controller::freeMemorySpace(const int) STUB std::ostream &operator <<(std::ostream &os, const StoreEntry &) { @@ -125,7 +125,7 @@ StoreEntry *storeGetPublicByRequest(HttpRequest * request, const KeyScope scope) STUB_RETVAL(NULL) StoreEntry *storeGetPublicByRequestMethod(HttpRequest * request, const HttpRequestMethod& method, const KeyScope scope) STUB_RETVAL(NULL) StoreEntry *storeCreateEntry(const char *, const char *, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL) -StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const RequestFlags &, const HttpRequestMethod&) STUB_RETVAL(NULL) +StoreEntry *storeCreatePureEntry(const char *storeId, const char *logUrl, const HttpRequestMethod&) STUB_RETVAL(nullptr) void storeConfigure(void) STUB int expiresMoreThan(time_t, time_t) STUB_RETVAL(0) void storeAppendPrintf(StoreEntry *, const char *,...) STUB diff -u -r -N squid-4.0.23/src/tests/stub_store_rebuild.cc squid-4.0.24/src/tests/stub_store_rebuild.cc --- squid-4.0.23/src/tests/stub_store_rebuild.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/stub_store_rebuild.cc 2018-03-08 02:16:46.000000000 +1300 @@ -19,7 +19,6 @@ #include "tests/STUB.h" void storeRebuildProgress(int sd_index, int total, int sofar) STUB -bool storeRebuildKeepEntry(const StoreEntry &tmpe, const cache_key *key, StoreRebuildData &counts) STUB_RETVAL(false) bool storeRebuildParseEntry(MemBuf &, StoreEntry &, cache_key *, StoreRebuildData &, uint64_t) STUB_RETVAL(false) void storeRebuildComplete(StoreRebuildData *) diff -u -r -N squid-4.0.23/src/tests/testRock.cc squid-4.0.24/src/tests/testRock.cc --- squid-4.0.23/src/tests/testRock.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/testRock.cc 2018-03-08 02:16:46.000000000 +1300 @@ -259,16 +259,28 @@ // try to swap out entry to a used unlocked slot { - StoreEntry *const pe = addEntry(4); + // without marking the old entry as deleted + StoreEntry *const pe = addEntry(3); - CPPUNIT_ASSERT_EQUAL(SWAPOUT_WRITING, pe->swap_status); - CPPUNIT_ASSERT_EQUAL(0, pe->swap_dirn); - CPPUNIT_ASSERT(pe->swap_filen >= 0); + CPPUNIT_ASSERT_EQUAL(SWAPOUT_NONE, pe->swap_status); + CPPUNIT_ASSERT_EQUAL(-1, pe->swap_dirn); + CPPUNIT_ASSERT_EQUAL(-1, pe->swap_filen); + pe->unlock("testRock::testRockSwapOut e#3"); + + // after marking the old entry as deleted + StoreEntry *const pe2 = getEntry(4); + CPPUNIT_ASSERT(pe2 != nullptr); + pe2->release(); + + StoreEntry *const pe3 = addEntry(4); + CPPUNIT_ASSERT_EQUAL(SWAPOUT_WRITING, pe3->swap_status); + CPPUNIT_ASSERT_EQUAL(0, pe3->swap_dirn); + CPPUNIT_ASSERT(pe3->swap_filen >= 0); StockEventLoop loop; loop.run(); - CPPUNIT_ASSERT_EQUAL(SWAPOUT_DONE, pe->swap_status); + CPPUNIT_ASSERT_EQUAL(SWAPOUT_DONE, pe3->swap_status); pe->unlock("testRock::testRockSwapOut e#4"); } diff -u -r -N squid-4.0.23/src/tests/testStoreController.cc squid-4.0.24/src/tests/testStoreController.cc --- squid-4.0.23/src/tests/testStoreController.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/testStoreController.cc 2018-03-08 02:16:46.000000000 +1300 @@ -111,8 +111,6 @@ e->expires = squid_curtime; e->lastModified(squid_curtime); e->refcount = 1; - EBIT_CLR(e->flags, RELEASE_REQUEST); - e->clearPrivate(); e->ping_status = PING_NONE; EBIT_CLR(e->flags, ENTRY_VALIDATED); e->hashInsert((const cache_key *)name.termedBuf()); /* do it after we clear KEY_PRIVATE */ diff -u -r -N squid-4.0.23/src/tests/testStoreHashIndex.cc squid-4.0.24/src/tests/testStoreHashIndex.cc --- squid-4.0.23/src/tests/testStoreHashIndex.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/testStoreHashIndex.cc 2018-03-08 02:16:46.000000000 +1300 @@ -89,8 +89,6 @@ e->expires = squid_curtime; e->lastModified(squid_curtime); e->refcount = 1; - EBIT_CLR(e->flags, RELEASE_REQUEST); - e->clearPrivate(); e->ping_status = PING_NONE; EBIT_CLR(e->flags, ENTRY_VALIDATED); e->hashInsert((const cache_key *)name.termedBuf()); /* do it after we clear KEY_PRIVATE */ diff -u -r -N squid-4.0.23/src/tests/TestSwapDir.h squid-4.0.24/src/tests/TestSwapDir.h --- squid-4.0.23/src/tests/TestSwapDir.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/tests/TestSwapDir.h 2018-03-08 02:16:46.000000000 +1300 @@ -24,7 +24,8 @@ virtual uint64_t currentSize() const override; virtual uint64_t currentCount() const override; virtual void stat(StoreEntry &) const override; - virtual void swappedOut(const StoreEntry &e) override {} + virtual void finalizeSwapoutSuccess(const StoreEntry &) override {} + virtual void finalizeSwapoutFailure(StoreEntry &) override {} virtual void reconfigure() override; virtual void init() override; virtual bool unlinkdUseful() const override; @@ -32,8 +33,9 @@ virtual StoreIOState::Pointer createStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override; virtual StoreIOState::Pointer openStoreIO(StoreEntry &, StoreIOState::STFNCB *, StoreIOState::STIOCB *, void *) override; virtual void parse(int, char*) override; - virtual void markForUnlink(StoreEntry &) override {} - virtual void unlink(StoreEntry &) override {} + virtual void evictCached(StoreEntry &) override {} + virtual void evictIfFound(const cache_key *) override {} + virtual bool hasReadableEntry(const StoreEntry &) const override { return false; } }; typedef RefCount TestSwapDirPointer; diff -u -r -N squid-4.0.23/src/Transients.cc squid-4.0.24/src/Transients.cc --- squid-4.0.23/src/Transients.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/Transients.cc 2018-03-08 02:16:46.000000000 +1300 @@ -26,8 +26,6 @@ /// shared memory segment path to use for Transients map static const SBuf MapLabel("transients_map"); -/// shared memory segment path to use for Transients map extras -static const char *ExtrasLabel = "transients_ex"; Transients::Transients(): map(NULL), locals(NULL) { @@ -50,8 +48,6 @@ map = new TransientsMap(MapLabel); map->cleaner = this; - extras = shm_old(TransientsMapExtras)(ExtrasLabel); - locals = new Locals(entryLimit, 0); } @@ -161,42 +157,16 @@ if (StoreEntry *oldE = locals->at(index)) { debugs(20, 3, "not joining private " << *oldE); assert(EBIT_TEST(oldE->flags, KEY_PRIVATE)); - } else if (StoreEntry *newE = copyFromShm(index)) { - return newE; // keep read lock to receive updates from others + map->closeForReading(index); + return nullptr; } - // private entry or loading failure - map->closeForReading(index); - return NULL; -} - -StoreEntry * -Transients::copyFromShm(const sfileno index) -{ - const TransientsMapExtras::Item &extra = extras->items[index]; - - // create a brand new store entry and initialize it with stored info - StoreEntry *e = storeCreatePureEntry(extra.url, extra.url, - extra.reqFlags, extra.reqMethod); - - assert(e->mem_obj); - e->mem_obj->method = extra.reqMethod; - e->mem_obj->xitTable.io = MemObject::ioReading; + StoreEntry *e = new StoreEntry(); + e->createMemObject(); e->mem_obj->xitTable.index = index; - - // TODO: Support collapsed revalidation for SMP-aware caches. - e->setPublicKey(ksDefault); - assert(e->key); - - // How do we know its SMP- and not just locally-collapsed? A worker gets - // locally-collapsed entries from the local store_table, not Transients. - // TODO: Can we remove smpCollapsed by not syncing non-transient entries? - e->mem_obj->smpCollapsed = true; - - assert(!locals->at(index)); - // We do not lock e because we do not want to prevent its destruction; - // e is tied to us via mem_obj so we will know when it is destructed. - locals->at(index) = e; + e->mem_obj->xitTable.io = Store::ioReading; + anchor->exportInto(*e); + // keep read lock to receive updates from others return e; } @@ -217,64 +187,49 @@ } void -Transients::startWriting(StoreEntry *e, const RequestFlags &reqFlags, - const HttpRequestMethod &reqMethod) +Transients::monitorIo(StoreEntry *e, const cache_key *key, const Store::IoStatus direction) { - assert(e); - assert(e->mem_obj); - assert(e->mem_obj->xitTable.index < 0); + assert(direction == Store::ioReading || direction == Store::ioWriting); - if (!map) { - debugs(20, 5, "No map to add " << *e); - return; + if (!e->hasTransients()) { + addEntry(e, key, direction); + e->mem_obj->xitTable.io = direction; } - sfileno index = 0; - Ipc::StoreMapAnchor *slot = map->openForWriting(reinterpret_cast(e->key), index); - if (!slot) { - debugs(20, 5, "collision registering " << *e); - return; - } - - try { - if (copyToShm(*e, index, reqFlags, reqMethod)) { - slot->set(*e); - e->mem_obj->xitTable.io = MemObject::ioWriting; - e->mem_obj->xitTable.index = index; - map->startAppending(index); - // keep write lock -- we will be supplying others with updates - return; - } - // fall through to the error handling code - } catch (const std::exception &x) { // TODO: should we catch ... as well? - debugs(20, 2, "error keeping entry " << index << - ' ' << *e << ": " << x.what()); - // fall through to the error handling code + assert(e->hasTransients()); + const auto index = e->mem_obj->xitTable.index; + if (const auto old = locals->at(index)) { + assert(old == e); + } else { + // We do not lock e because we do not want to prevent its destruction; + // e is tied to us via mem_obj so we will know when it is destructed. + locals->at(index) = e; } - - map->abortWriting(index); } -/// copies all relevant local data to shared memory -bool -Transients::copyToShm(const StoreEntry &e, const sfileno index, - const RequestFlags &reqFlags, - const HttpRequestMethod &reqMethod) +/// creates a new Transients entry or throws +void +Transients::addEntry(StoreEntry *e, const cache_key *key, const Store::IoStatus direction) { - TransientsMapExtras::Item &extra = extras->items[index]; - - const char *url = e.url(); - const size_t urlLen = strlen(url); - Must(urlLen < sizeof(extra.url)); // we have space to store it all, plus 0 - strncpy(extra.url, url, sizeof(extra.url)); - extra.url[urlLen] = '\0'; + assert(e); + assert(e->mem_obj); + assert(!e->hasTransients()); - extra.reqFlags = reqFlags; + Must(map); // configured to track transients - Must(reqMethod != Http::METHOD_OTHER); - extra.reqMethod = reqMethod.id(); + sfileno index = 0; + Ipc::StoreMapAnchor *slot = map->openForWriting(key, index); + Must(slot); // no writer collisions - return true; + slot->set(*e, key); + e->mem_obj->xitTable.index = index; + if (direction == Store::ioWriting) { + // keep write lock; the caller will decide what to do with it + map->startAppending(e->mem_obj->xitTable.index); + } else { + // keep the entry locked (for reading) to receive remote DELETE events + map->closeForWriting(e->mem_obj->xitTable.index); + } } void @@ -284,49 +239,30 @@ } void -Transients::abandon(const StoreEntry &e) -{ - assert(e.mem_obj && map); - map->freeEntry(e.mem_obj->xitTable.index); // just marks the locked entry - CollapsedForwarding::Broadcast(e); - // We do not unlock the entry now because the problem is most likely with - // the server resource rather than a specific cache writer, so we want to - // prevent other readers from collapsing requests for that resource. -} - -bool -Transients::abandoned(const StoreEntry &e) const -{ - assert(e.mem_obj); - return abandonedAt(e.mem_obj->xitTable.index); -} - -/// whether an in-transit entry at the index is now abandoned by its writer -bool -Transients::abandonedAt(const sfileno index) const +Transients::status(const StoreEntry &entry, bool &aborted, bool &waitingToBeFreed) const { assert(map); - return map->readableEntry(index).waitingToBeFreed; + assert(entry.hasTransients()); + const auto idx = entry.mem_obj->xitTable.index; + const auto &anchor = isWriter(entry) ? + map->writeableEntry(idx) : map->readableEntry(idx); + aborted = anchor.writerHalted; + waitingToBeFreed = anchor.waitingToBeFreed; } void Transients::completeWriting(const StoreEntry &e) { - if (e.mem_obj && e.mem_obj->xitTable.index >= 0) { - assert(e.mem_obj->xitTable.io == MemObject::ioWriting); - // there will be no more updates from us after this, so we must prevent - // future readers from joining - map->freeEntry(e.mem_obj->xitTable.index); // just marks the locked entry - map->closeForWriting(e.mem_obj->xitTable.index); - e.mem_obj->xitTable.index = -1; - e.mem_obj->xitTable.io = MemObject::ioDone; - } + assert(e.hasTransients()); + assert(isWriter(e)); + map->closeForWriting(e.mem_obj->xitTable.index, true); + e.mem_obj->xitTable.io = Store::ioReading; } int Transients::readers(const StoreEntry &e) const { - if (e.mem_obj && e.mem_obj->xitTable.index >= 0) { + if (e.hasTransients()) { assert(map); return map->peekAtEntry(e.mem_obj->xitTable.index).lock.readers; } @@ -334,32 +270,46 @@ } void -Transients::markForUnlink(StoreEntry &e) +Transients::evictCached(StoreEntry &e) { - unlink(e); + debugs(20, 5, e); + if (e.hasTransients()) { + const auto index = e.mem_obj->xitTable.index; + if (map->freeEntry(index)) { + // Delay syncCollapsed(index) which may end `e` wait for updates. + // Calling it directly/here creates complex reentrant call chains. + CollapsedForwarding::Broadcast(e, true); + } + } // else nothing to do because e must be private } void -Transients::unlink(StoreEntry &e) +Transients::evictIfFound(const cache_key *key) { - if (e.mem_obj && e.mem_obj->xitTable.io == MemObject::ioWriting) - abandon(e); + if (!map) + return; + + const sfileno index = map->fileNoByKey(key); + if (map->freeEntry(index)) + CollapsedForwarding::Broadcast(index, true); } void -Transients::disconnect(MemObject &mem_obj) +Transients::disconnect(StoreEntry &entry) { - if (mem_obj.xitTable.index >= 0) { + debugs(20, 5, entry); + if (entry.hasTransients()) { + auto &xitTable = entry.mem_obj->xitTable; assert(map); - if (mem_obj.xitTable.io == MemObject::ioWriting) { - map->abortWriting(mem_obj.xitTable.index); + if (isWriter(entry)) { + map->abortWriting(xitTable.index); } else { - assert(mem_obj.xitTable.io == MemObject::ioReading); - map->closeForReading(mem_obj.xitTable.index); + assert(isReader(entry)); + map->closeForReading(xitTable.index); } - locals->at(mem_obj.xitTable.index) = NULL; - mem_obj.xitTable.index = -1; - mem_obj.xitTable.io = MemObject::ioDone; + locals->at(xitTable.index) = nullptr; + xitTable.index = -1; + xitTable.io = Store::ioDone; } } @@ -374,12 +324,30 @@ return Config.collapsed_forwarding_shared_entries_limit; } +bool +Transients::markedForDeletion(const cache_key *key) const +{ + assert(map); + return map->markedForDeletion(key); +} + +bool +Transients::isReader(const StoreEntry &e) const +{ + return e.mem_obj && e.mem_obj->xitTable.io == Store::ioReading; +} + +bool +Transients::isWriter(const StoreEntry &e) const +{ + return e.mem_obj && e.mem_obj->xitTable.io == Store::ioWriting; +} + /// initializes shared memory segment used by Transients class TransientsRr: public Ipc::Mem::RegisteredRunner { public: /* RegisteredRunner API */ - TransientsRr(): mapOwner(NULL), extrasOwner(NULL) {} virtual void useConfig(); virtual ~TransientsRr(); @@ -387,8 +355,7 @@ virtual void create(); private: - TransientsMap::Owner *mapOwner; - Ipc::Mem::Owner *extrasOwner; + TransientsMap::Owner *mapOwner = nullptr; }; RunnerRegistrationEntry(TransientsRr); @@ -412,13 +379,10 @@ Must(!mapOwner); mapOwner = TransientsMap::Init(MapLabel, entryLimit); - Must(!extrasOwner); - extrasOwner = shm_new(TransientsMapExtras)(ExtrasLabel, entryLimit); } TransientsRr::~TransientsRr() { - delete extrasOwner; delete mapOwner; } diff -u -r -N squid-4.0.23/src/Transients.h squid-4.0.24/src/Transients.h --- squid-4.0.23/src/Transients.h 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/Transients.h 2018-03-08 02:16:46.000000000 +1300 @@ -9,26 +9,20 @@ #ifndef SQUID_TRANSIENTS_H #define SQUID_TRANSIENTS_H -#include "http/MethodType.h" #include "ipc/mem/Page.h" #include "ipc/mem/PageStack.h" #include "ipc/StoreMap.h" #include "Store.h" #include "store/Controlled.h" +#include "store/forward.h" #include -// StoreEntry restoration info not already stored by Ipc::StoreMap -struct TransientsMapExtraItem { - char url[MAX_URL+1]; ///< Request-URI; TODO: decrease MAX_URL by one - RequestFlags reqFlags; ///< request flags - Http::MethodType reqMethod; ///< request method; extensions are not supported -}; -typedef Ipc::StoreMapItems TransientsMapExtras; typedef Ipc::StoreMap TransientsMap; /// Keeps track of store entries being delivered to clients that arrived before -/// those entries were [fully] cached. This shared table is necessary to sync -/// the entry-writing worker with entry-reading worker(s). +/// those entries were [fully] cached. This SMP-shared table is necessary to +/// * sync an entry-writing worker with entry-reading worker(s); and +/// * sync an entry-deleting worker with both entry-reading/writing workers. class Transients: public Store::Controlled, public Ipc::StoreMapCleaner { public: @@ -38,23 +32,23 @@ /// return a local, previously collapsed entry StoreEntry *findCollapsed(const sfileno xitIndex); - /// add an in-transit entry suitable for collapsing future requests - void startWriting(StoreEntry *e, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod); + /// start listening for remote DELETE requests targeting either a complete + /// StoreEntry (ioReading) or a being-formed miss StoreEntry (ioWriting) + void monitorIo(StoreEntry*, const cache_key*, const Store::IoStatus); /// called when the in-transit entry has been successfully cached void completeWriting(const StoreEntry &e); - /// the calling entry writer no longer expects to cache this entry - void abandon(const StoreEntry &e); - - /// whether an in-transit entry is now abandoned by its writer - bool abandoned(const StoreEntry &e) const; + /// copies current shared entry metadata into parameters + /// \param aborted whether the entry was aborted + /// \param waitingToBeFreed whether the entry was marked for deletion + void status(const StoreEntry &e, bool &aborted, bool &waitingToBeFreed) const; /// number of entry readers some time ago int readers(const StoreEntry &e) const; - /// the caller is done writing or reading this entry - void disconnect(MemObject &mem_obj); + /// the caller is done writing or reading the given entry + void disconnect(StoreEntry &); /* Store API */ virtual StoreEntry *get(const cache_key *) override; @@ -69,18 +63,24 @@ virtual void stat(StoreEntry &e) const override; virtual void reference(StoreEntry &e) override; virtual bool dereference(StoreEntry &e) override; - virtual void markForUnlink(StoreEntry &e) override; - virtual void unlink(StoreEntry &e) override; + virtual void evictCached(StoreEntry &) override; + virtual void evictIfFound(const cache_key *) override; virtual void maintain() override; virtual bool smpAware() const override { return true; } + /// Whether an entry with the given public key exists and (but) was + /// marked for removal some time ago; get(key) returns nil in such cases. + bool markedForDeletion(const cache_key *) const; + + /// whether the entry is in "reading from Transients" I/O state + bool isReader(const StoreEntry &) const; + /// whether the entry is in "writing to Transients" I/O state + bool isWriter(const StoreEntry &) const; + static int64_t EntryLimit(); protected: - StoreEntry *copyFromShm(const sfileno index); - bool copyToShm(const StoreEntry &e, const sfileno index, const RequestFlags &reqFlags, const HttpRequestMethod &reqMethod); - - bool abandonedAt(const sfileno index) const; + void addEntry(StoreEntry*, const cache_key *, const Store::IoStatus); // Ipc::StoreMapCleaner API virtual void noteFreeMapSlice(const Ipc::StoreMapSliceId sliceId) override; @@ -89,12 +89,9 @@ /// shared packed info indexed by Store keys, for creating new StoreEntries TransientsMap *map; - /// shared packed info that standard StoreMap does not store for us - typedef TransientsMapExtras Extras; - Ipc::Mem::Pointer extras; - typedef std::vector Locals; - /// local collapsed entries indexed by transient ID, for syncing old StoreEntries + /// local collapsed reader and writer entries, indexed by transient ID, + /// for syncing old StoreEntries Locals *locals; }; diff -u -r -N squid-4.0.23/src/whois.cc squid-4.0.24/src/whois.cc --- squid-4.0.23/src/whois.cc 2018-01-20 02:28:39.000000000 +1300 +++ squid-4.0.24/src/whois.cc 2018-03-08 02:16:46.000000000 +1300 @@ -158,7 +158,8 @@ entry->timestampsSet(); entry->flush(); - entry->makePublic(); + if (!entry->makePublic()) + entry->makePrivate(true); fwd->complete(); debugs(75, 3, "whoisReadReply: Done: " << entry->url()); diff -u -r -N squid-4.0.23/tools/helper-mux/helper-mux.8 squid-4.0.24/tools/helper-mux/helper-mux.8 --- squid-4.0.23/tools/helper-mux/helper-mux.8 2018-01-20 02:38:25.000000000 +1300 +++ squid-4.0.24/tools/helper-mux/helper-mux.8 2018-03-08 02:33:27.000000000 +1300 @@ -129,7 +129,7 @@ .\" ======================================================================== .\" .IX Title "HELPER-MUX 8" -.TH HELPER-MUX 8 "2018-01-19" "perl v5.26.1" "User Contributed Perl Documentation" +.TH HELPER-MUX 8 "2018-03-07" "perl v5.26.1" "User Contributed Perl Documentation" .\" For nroff, turn off justification. Always turn off hyphenation; it makes .\" way too many mistakes in technical documents. .if n .ad l diff -u -r -N squid-4.0.23/tools/MemBuf.cc squid-4.0.24/tools/MemBuf.cc --- squid-4.0.23/tools/MemBuf.cc 2018-01-20 02:38:25.000000000 +1300 +++ squid-4.0.24/tools/MemBuf.cc 2018-03-08 02:33:27.000000000 +1300 @@ -76,15 +76,6 @@ #include "MemBuf.h" #include "profiler/Profiler.h" -#ifdef VA_COPY -#undef VA_COPY -#endif -#if defined HAVE_VA_COPY -#define VA_COPY va_copy -#elif defined HAVE___VA_COPY -#define VA_COPY __va_copy -#endif - /* local constants */ /* default values for buffer sizes, used by memBufDefInit */ @@ -268,10 +259,6 @@ void MemBuf::vappendf(const char *fmt, va_list vargs) { -#ifdef VA_COPY - va_list ap; -#endif - int sz = 0; assert(fmt); assert(buf); @@ -282,18 +269,15 @@ mb_size_t free_space = capacity - size; /* put as much as we can */ -#ifdef VA_COPY /* Fix of bug 753r. The value of vargs is undefined * after vsnprintf() returns. Make a copy of vargs * incase we loop around and call vsnprintf() again. */ - VA_COPY(ap,vargs); + va_list ap; + va_copy(ap,vargs); sz = vsnprintf(buf + size, free_space, fmt, ap); va_end(ap); -#else /* VA_COPY */ - sz = vsnprintf(buf + size, free_space, fmt, vargs); -#endif /*VA_COPY*/ /* check for possible overflow */ /* snprintf on Linuz returns -1 on overflows */ /* snprintf on FreeBSD returns at least free_space on overflows */