Merge pull request #85 from tomav/move-test-framework-to-bats

Fixes #84 => Move test framework from assert.sh to bats
This commit is contained in:
Thomas VIAL 2016-02-25 09:42:09 +01:00
commit 67f3381808
10 changed files with 1104 additions and 303 deletions

View file

@ -41,7 +41,7 @@ fixtures:
tests:
# Start tests
/bin/bash ./test/test.sh
./test/bats/bats test/tests.bats
clean:
# Get default files back

186
assert.sh
View file

@ -1,186 +0,0 @@
#!/bin/bash
# assert.sh 1.1 - bash unit testing framework
# Copyright (C) 2009-2015 Robert Lehmann
#
# http://github.com/lehmannro/assert.sh
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
export DISCOVERONLY=${DISCOVERONLY:-}
export DEBUG=${DEBUG:-}
export STOP=${STOP:-}
export INVARIANT=${INVARIANT:-}
export CONTINUE=${CONTINUE:-}
args="$(getopt -n "$0" -l \
verbose,help,stop,discover,invariant,continue vhxdic $*)" \
|| exit -1
for arg in $args; do
case "$arg" in
-h)
echo "$0 [-vxidc]" \
"[--verbose] [--stop] [--invariant] [--discover] [--continue]"
echo "`sed 's/./ /g' <<< "$0"` [-h] [--help]"
exit 0;;
--help)
cat <<EOF
Usage: $0 [options]
Language-agnostic unit tests for subprocesses.
Options:
-v, --verbose generate output for every individual test case
-x, --stop stop running tests after the first failure
-i, --invariant do not measure timings to remain invariant between runs
-d, --discover collect test suites only, do not run any tests
-c, --continue do not modify exit code to test suite status
-h show brief usage information and exit
--help show this help message and exit
EOF
exit 0;;
-v|--verbose)
DEBUG=1;;
-x|--stop)
STOP=1;;
-i|--invariant)
INVARIANT=1;;
-d|--discover)
DISCOVERONLY=1;;
-c|--continue)
CONTINUE=1;;
esac
done
_indent=$'\n\t' # local format helper
_assert_reset() {
tests_ran=0
tests_failed=0
tests_errors=()
tests_starttime="$(date +%s%N)" # nanoseconds_since_epoch
}
assert_end() {
# assert_end [suite ..]
tests_endtime="$(date +%s%N)"
# required visible decimal place for seconds (leading zeros if needed)
local tests_time="$( \
printf "%010d" "$(( ${tests_endtime/%N/000000000}
- ${tests_starttime/%N/000000000} ))")" # in ns
tests="$tests_ran ${*:+$* }tests"
[[ -n "$DISCOVERONLY" ]] && echo "collected $tests." && _assert_reset && return
[[ -n "$DEBUG" ]] && echo
# to get report_time split tests_time on 2 substrings:
# ${tests_time:0:${#tests_time}-9} - seconds
# ${tests_time:${#tests_time}-9:3} - milliseconds
[[ -z "$INVARIANT" ]] \
&& report_time=" in ${tests_time:0:${#tests_time}-9}.${tests_time:${#tests_time}-9:3}s" \
|| report_time=
if [[ "$tests_failed" -eq 0 ]]; then
echo "all $tests passed$report_time."
else
for error in "${tests_errors[@]}"; do echo "$error"; done
echo "$tests_failed of $tests failed$report_time."
fi
tests_failed_previous=$tests_failed
[[ $tests_failed -gt 0 ]] && tests_suite_status=1
_assert_reset
}
assert() {
# assert <command> <expected stdout> [stdin]
(( tests_ran++ )) || :
[[ -z "$DISCOVERONLY" ]] || return
expected=$(echo -ne "${2:-}")
result="$(eval 2>/dev/null $1 <<< ${3:-})" || true
if [[ "$result" == "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
return
fi
result="$(sed -e :a -e '$!N;s/\n/\\n/;ta' <<< "$result")"
[[ -z "$result" ]] && result="nothing" || result="\"$result\""
[[ -z "$2" ]] && expected="nothing" || expected="\"$2\""
_assert_fail "expected $expected${_indent}got $result" "$1" "$3"
}
assert_raises() {
# assert_raises <command> <expected code> [stdin]
(( tests_ran++ )) || :
[[ -z "$DISCOVERONLY" ]] || return
status=0
(eval $1 <<< ${3:-}) > /dev/null 2>&1 || status=$?
expected=${2:-0}
if [[ "$status" -eq "$expected" ]]; then
[[ -z "$DEBUG" ]] || echo -n .
return
fi
_assert_fail "program terminated with code $status instead of $expected" "$1" "$3"
}
_assert_fail() {
# _assert_fail <failure> <command> <stdin>
[[ -n "$DEBUG" ]] && echo -n X
report="test #$tests_ran \"$2${3:+ <<< $3}\" failed:${_indent}$1"
if [[ -n "$STOP" ]]; then
[[ -n "$DEBUG" ]] && echo
echo "$report"
exit 1
fi
tests_errors[$tests_failed]="$report"
(( tests_failed++ )) || :
}
skip_if() {
# skip_if <command ..>
(eval $@) > /dev/null 2>&1 && status=0 || status=$?
[[ "$status" -eq 0 ]] || return
skip
}
skip() {
# skip (no arguments)
shopt -q extdebug && tests_extdebug=0 || tests_extdebug=1
shopt -q -o errexit && tests_errexit=0 || tests_errexit=1
# enable extdebug so returning 1 in a DEBUG trap handler skips next command
shopt -s extdebug
# disable errexit (set -e) so we can safely return 1 without causing exit
set +o errexit
tests_trapped=0
trap _skip DEBUG
}
_skip() {
if [[ $tests_trapped -eq 0 ]]; then
# DEBUG trap for command we want to skip. Do not remove the handler
# yet because *after* the command we need to reset extdebug/errexit (in
# another DEBUG trap.)
tests_trapped=1
[[ -z "$DEBUG" ]] || echo -n s
return 1
else
trap - DEBUG
[[ $tests_extdebug -eq 0 ]] || shopt -u extdebug
[[ $tests_errexit -eq 1 ]] || set -o errexit
return 0
fi
}
_assert_reset
: ${tests_suite_status:=0} # remember if any of the tests failed so far
_assert_cleanup() {
local status=$?
# modify exit code if it's not already non-zero
[[ $status -eq 0 && -z $CONTINUE ]] && exit $tests_suite_status
}
trap _assert_cleanup EXIT

View file

@ -0,0 +1,3 @@
EHLO mail
AUTH PLAIN WRONGPASSWORD
QUIT

142
test/bats/bats Executable file
View file

@ -0,0 +1,142 @@
#!/usr/bin/env bash
set -e
version() {
echo "Bats 0.4.0"
}
usage() {
version
echo "Usage: bats [-c] [-p | -t] <test> [<test> ...]"
}
help() {
usage
echo
echo " <test> is the path to a Bats test file, or the path to a directory"
echo " containing Bats test files."
echo
echo " -c, --count Count the number of test cases without running any tests"
echo " -h, --help Display this help message"
echo " -p, --pretty Show results in pretty format (default for terminals)"
echo " -t, --tap Show results in TAP format"
echo " -v, --version Display the version number"
echo
echo " For more information, see https://github.com/sstephenson/bats"
echo
}
resolve_link() {
$(type -p greadlink readlink | head -1) "$1"
}
abs_dirname() {
local cwd="$(pwd)"
local path="$1"
while [ -n "$path" ]; do
cd "${path%/*}"
local name="${path##*/}"
path="$(resolve_link "$name" || true)"
done
pwd
cd "$cwd"
}
expand_path() {
{ cd "$(dirname "$1")" 2>/dev/null
local dirname="$PWD"
cd "$OLDPWD"
echo "$dirname/$(basename "$1")"
} || echo "$1"
}
BATS_LIBEXEC="$(abs_dirname "$0")"
export BATS_PREFIX="$(abs_dirname "$BATS_LIBEXEC")"
export BATS_CWD="$(abs_dirname .)"
export PATH="$BATS_LIBEXEC:$PATH"
options=()
arguments=()
for arg in "$@"; do
if [ "${arg:0:1}" = "-" ]; then
if [ "${arg:1:1}" = "-" ]; then
options[${#options[*]}]="${arg:2}"
else
index=1
while option="${arg:$index:1}"; do
[ -n "$option" ] || break
options[${#options[*]}]="$option"
let index+=1
done
fi
else
arguments[${#arguments[*]}]="$arg"
fi
done
unset count_flag pretty
[ -t 0 ] && [ -t 1 ] && pretty="1"
[ -n "$CI" ] && pretty=""
for option in "${options[@]}"; do
case "$option" in
"h" | "help" )
help
exit 0
;;
"v" | "version" )
version
exit 0
;;
"c" | "count" )
count_flag="-c"
;;
"t" | "tap" )
pretty=""
;;
"p" | "pretty" )
pretty="1"
;;
* )
usage >&2
exit 1
;;
esac
done
if [ "${#arguments[@]}" -eq 0 ]; then
usage >&2
exit 1
fi
filenames=()
for filename in "${arguments[@]}"; do
if [ -d "$filename" ]; then
shopt -s nullglob
for suite_filename in "$(expand_path "$filename")"/*.bats; do
filenames["${#filenames[@]}"]="$suite_filename"
done
shopt -u nullglob
else
filenames["${#filenames[@]}"]="$(expand_path "$filename")"
fi
done
if [ "${#filenames[@]}" -eq 1 ]; then
command="bats-exec-test"
else
command="bats-exec-suite"
fi
if [ -n "$pretty" ]; then
extended_syntax_flag="-x"
formatter="bats-format-tap-stream"
else
extended_syntax_flag=""
formatter="cat"
fi
set -o pipefail execfail
exec "$command" $count_flag $extended_syntax_flag "${filenames[@]}" | "$formatter"

55
test/bats/bats-exec-suite Executable file
View file

@ -0,0 +1,55 @@
#!/usr/bin/env bash
set -e
count_only_flag=""
if [ "$1" = "-c" ]; then
count_only_flag=1
shift
fi
extended_syntax_flag=""
if [ "$1" = "-x" ]; then
extended_syntax_flag="-x"
shift
fi
trap "kill 0; exit 1" int
count=0
for filename in "$@"; do
let count+="$(bats-exec-test -c "$filename")"
done
if [ -n "$count_only_flag" ]; then
echo "$count"
exit
fi
echo "1..$count"
status=0
offset=0
for filename in "$@"; do
index=0
{
IFS= read -r # 1..n
while IFS= read -r line; do
case "$line" in
"begin "* )
let index+=1
echo "${line/ $index / $(($offset + $index)) }"
;;
"ok "* | "not ok "* )
[ -n "$extended_syntax_flag" ] || let index+=1
echo "${line/ $index / $(($offset + $index)) }"
[ "${line:0:6}" != "not ok" ] || status=1
;;
* )
echo "$line"
;;
esac
done
} < <( bats-exec-test $extended_syntax_flag "$filename" )
offset=$(($offset + $index))
done
exit "$status"

346
test/bats/bats-exec-test Executable file
View file

@ -0,0 +1,346 @@
#!/usr/bin/env bash
set -e
set -E
set -T
BATS_COUNT_ONLY=""
if [ "$1" = "-c" ]; then
BATS_COUNT_ONLY=1
shift
fi
BATS_EXTENDED_SYNTAX=""
if [ "$1" = "-x" ]; then
BATS_EXTENDED_SYNTAX="$1"
shift
fi
BATS_TEST_FILENAME="$1"
if [ -z "$BATS_TEST_FILENAME" ]; then
echo "usage: bats-exec <filename>" >&2
exit 1
elif [ ! -f "$BATS_TEST_FILENAME" ]; then
echo "bats: $BATS_TEST_FILENAME does not exist" >&2
exit 1
else
shift
fi
BATS_TEST_DIRNAME="$(dirname "$BATS_TEST_FILENAME")"
BATS_TEST_NAMES=()
load() {
local name="$1"
local filename
if [ "${name:0:1}" = "/" ]; then
filename="${name}"
else
filename="$BATS_TEST_DIRNAME/${name}.bash"
fi
[ -f "$filename" ] || {
echo "bats: $filename does not exist" >&2
exit 1
}
source "${filename}"
}
run() {
local e E T oldIFS
[[ ! "$-" =~ e ]] || e=1
[[ ! "$-" =~ E ]] || E=1
[[ ! "$-" =~ T ]] || T=1
set +e
set +E
set +T
output="$("$@" 2>&1)"
status="$?"
oldIFS=$IFS
IFS=$'\n' lines=($output)
[ -z "$e" ] || set -e
[ -z "$E" ] || set -E
[ -z "$T" ] || set -T
IFS=$oldIFS
}
setup() {
true
}
teardown() {
true
}
skip() {
BATS_TEST_SKIPPED=${1:-1}
BATS_TEST_COMPLETED=1
exit 0
}
bats_test_begin() {
BATS_TEST_DESCRIPTION="$1"
if [ -n "$BATS_EXTENDED_SYNTAX" ]; then
echo "begin $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3
fi
setup
}
bats_test_function() {
local test_name="$1"
BATS_TEST_NAMES["${#BATS_TEST_NAMES[@]}"]="$test_name"
}
bats_capture_stack_trace() {
BATS_PREVIOUS_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" )
BATS_CURRENT_STACK_TRACE=()
local test_pattern=" $BATS_TEST_NAME $BATS_TEST_SOURCE"
local setup_pattern=" setup $BATS_TEST_SOURCE"
local teardown_pattern=" teardown $BATS_TEST_SOURCE"
local frame
local index=1
while frame="$(caller "$index")"; do
BATS_CURRENT_STACK_TRACE["${#BATS_CURRENT_STACK_TRACE[@]}"]="$frame"
if [[ "$frame" = *"$test_pattern" || \
"$frame" = *"$setup_pattern" || \
"$frame" = *"$teardown_pattern" ]]; then
break
else
let index+=1
fi
done
BATS_SOURCE="$(bats_frame_filename "${BATS_CURRENT_STACK_TRACE[0]}")"
BATS_LINENO="$(bats_frame_lineno "${BATS_CURRENT_STACK_TRACE[0]}")"
}
bats_print_stack_trace() {
local frame
local index=1
local count="${#@}"
for frame in "$@"; do
local filename="$(bats_trim_filename "$(bats_frame_filename "$frame")")"
local lineno="$(bats_frame_lineno "$frame")"
if [ $index -eq 1 ]; then
echo -n "# ("
else
echo -n "# "
fi
local fn="$(bats_frame_function "$frame")"
if [ "$fn" != "$BATS_TEST_NAME" ]; then
echo -n "from function \`$fn' "
fi
if [ $index -eq $count ]; then
echo "in test file $filename, line $lineno)"
else
echo "in file $filename, line $lineno,"
fi
let index+=1
done
}
bats_print_failed_command() {
local frame="$1"
local status="$2"
local filename="$(bats_frame_filename "$frame")"
local lineno="$(bats_frame_lineno "$frame")"
local failed_line="$(bats_extract_line "$filename" "$lineno")"
local failed_command="$(bats_strip_string "$failed_line")"
echo -n "# \`${failed_command}' "
if [ $status -eq 1 ]; then
echo "failed"
else
echo "failed with status $status"
fi
}
bats_frame_lineno() {
local frame="$1"
local lineno="${frame%% *}"
echo "$lineno"
}
bats_frame_function() {
local frame="$1"
local rest="${frame#* }"
local fn="${rest%% *}"
echo "$fn"
}
bats_frame_filename() {
local frame="$1"
local rest="${frame#* }"
local filename="${rest#* }"
if [ "$filename" = "$BATS_TEST_SOURCE" ]; then
echo "$BATS_TEST_FILENAME"
else
echo "$filename"
fi
}
bats_extract_line() {
local filename="$1"
local lineno="$2"
sed -n "${lineno}p" "$filename"
}
bats_strip_string() {
local string="$1"
printf "%s" "$string" | sed -e "s/^[ "$'\t'"]*//" -e "s/[ "$'\t'"]*$//"
}
bats_trim_filename() {
local filename="$1"
local length="${#BATS_CWD}"
if [ "${filename:0:length+1}" = "${BATS_CWD}/" ]; then
echo "${filename:length+1}"
else
echo "$filename"
fi
}
bats_debug_trap() {
if [ "$BASH_SOURCE" != "$1" ]; then
bats_capture_stack_trace
fi
}
bats_error_trap() {
BATS_ERROR_STATUS="$?"
BATS_ERROR_STACK_TRACE=( "${BATS_PREVIOUS_STACK_TRACE[@]}" )
trap - debug
}
bats_teardown_trap() {
trap "bats_exit_trap" exit
local status=0
teardown >>"$BATS_OUT" 2>&1 || status="$?"
if [ $status -eq 0 ]; then
BATS_TEARDOWN_COMPLETED=1
elif [ -n "$BATS_TEST_COMPLETED" ]; then
BATS_ERROR_STATUS="$status"
BATS_ERROR_STACK_TRACE=( "${BATS_CURRENT_STACK_TRACE[@]}" )
fi
bats_exit_trap
}
bats_exit_trap() {
local status
local skipped
trap - err exit
skipped=""
if [ -n "$BATS_TEST_SKIPPED" ]; then
skipped=" # skip"
if [ "1" != "$BATS_TEST_SKIPPED" ]; then
skipped+=" ($BATS_TEST_SKIPPED)"
fi
fi
if [ -z "$BATS_TEST_COMPLETED" ] || [ -z "$BATS_TEARDOWN_COMPLETED" ]; then
echo "not ok $BATS_TEST_NUMBER $BATS_TEST_DESCRIPTION" >&3
bats_print_stack_trace "${BATS_ERROR_STACK_TRACE[@]}" >&3
bats_print_failed_command "${BATS_ERROR_STACK_TRACE[${#BATS_ERROR_STACK_TRACE[@]}-1]}" "$BATS_ERROR_STATUS" >&3
sed -e "s/^/# /" < "$BATS_OUT" >&3
status=1
else
echo "ok ${BATS_TEST_NUMBER}${skipped} ${BATS_TEST_DESCRIPTION}" >&3
status=0
fi
rm -f "$BATS_OUT"
exit "$status"
}
bats_perform_tests() {
echo "1..$#"
test_number=1
status=0
for test_name in "$@"; do
"$0" $BATS_EXTENDED_SYNTAX "$BATS_TEST_FILENAME" "$test_name" "$test_number" || status=1
let test_number+=1
done
exit "$status"
}
bats_perform_test() {
BATS_TEST_NAME="$1"
if [ "$(type -t "$BATS_TEST_NAME" || true)" = "function" ]; then
BATS_TEST_NUMBER="$2"
if [ -z "$BATS_TEST_NUMBER" ]; then
echo "1..1"
BATS_TEST_NUMBER="1"
fi
BATS_TEST_COMPLETED=""
BATS_TEARDOWN_COMPLETED=""
trap "bats_debug_trap \"\$BASH_SOURCE\"" debug
trap "bats_error_trap" err
trap "bats_teardown_trap" exit
"$BATS_TEST_NAME" >>"$BATS_OUT" 2>&1
BATS_TEST_COMPLETED=1
else
echo "bats: unknown test name \`$BATS_TEST_NAME'" >&2
exit 1
fi
}
if [ -z "$TMPDIR" ]; then
BATS_TMPDIR="/tmp"
else
BATS_TMPDIR="${TMPDIR%/}"
fi
BATS_TMPNAME="$BATS_TMPDIR/bats.$$"
BATS_PARENT_TMPNAME="$BATS_TMPDIR/bats.$PPID"
BATS_OUT="${BATS_TMPNAME}.out"
bats_preprocess_source() {
BATS_TEST_SOURCE="${BATS_TMPNAME}.src"
{ tr -d '\r' < "$BATS_TEST_FILENAME"; echo; } | bats-preprocess > "$BATS_TEST_SOURCE"
trap "bats_cleanup_preprocessed_source" err exit
trap "bats_cleanup_preprocessed_source; exit 1" int
}
bats_cleanup_preprocessed_source() {
rm -f "$BATS_TEST_SOURCE"
}
bats_evaluate_preprocessed_source() {
if [ -z "$BATS_TEST_SOURCE" ]; then
BATS_TEST_SOURCE="${BATS_PARENT_TMPNAME}.src"
fi
source "$BATS_TEST_SOURCE"
}
exec 3<&1
if [ "$#" -eq 0 ]; then
bats_preprocess_source
bats_evaluate_preprocessed_source
if [ -n "$BATS_COUNT_ONLY" ]; then
echo "${#BATS_TEST_NAMES[@]}"
else
bats_perform_tests "${BATS_TEST_NAMES[@]}"
fi
else
bats_evaluate_preprocessed_source
bats_perform_test "$@"
fi

165
test/bats/bats-format-tap-stream Executable file
View file

@ -0,0 +1,165 @@
#!/usr/bin/env bash
set -e
# Just stream the TAP output (sans extended syntax) if tput is missing
command -v tput >/dev/null || exec grep -v "^begin "
header_pattern='[0-9]+\.\.[0-9]+'
IFS= read -r header
if [[ "$header" =~ $header_pattern ]]; then
count="${header:3}"
index=0
failures=0
skipped=0
name=""
count_column_width=$(( ${#count} * 2 + 2 ))
else
# If the first line isn't a TAP plan, print it and pass the rest through
printf "%s\n" "$header"
exec cat
fi
update_screen_width() {
screen_width="$(tput cols)"
count_column_left=$(( $screen_width - $count_column_width ))
}
trap update_screen_width WINCH
update_screen_width
begin() {
go_to_column 0
printf_with_truncation $(( $count_column_left - 1 )) " %s" "$name"
clear_to_end_of_line
go_to_column $count_column_left
printf "%${#count}s/${count}" "$index"
go_to_column 1
}
pass() {
go_to_column 0
printf " ✓ %s" "$name"
advance
}
skip() {
local reason="$1"
[ -z "$reason" ] || reason=": $reason"
go_to_column 0
printf " - %s (skipped%s)" "$name" "$reason"
advance
}
fail() {
go_to_column 0
set_color 1 bold
printf " ✗ %s" "$name"
advance
}
log() {
set_color 1
printf " %s\n" "$1"
clear_color
}
summary() {
printf "\n%d test%s" "$count" "$(plural "$count")"
printf ", %d failure%s" "$failures" "$(plural "$failures")"
if [ "$skipped" -gt 0 ]; then
printf ", %d skipped" "$skipped"
fi
printf "\n"
}
printf_with_truncation() {
local width="$1"
shift
local string="$(printf "$@")"
if [ "${#string}" -gt "$width" ]; then
printf "%s..." "${string:0:$(( $width - 4 ))}"
else
printf "%s" "$string"
fi
}
go_to_column() {
local column="$1"
printf "\x1B[%dG" $(( $column + 1 ))
}
clear_to_end_of_line() {
printf "\x1B[K"
}
advance() {
clear_to_end_of_line
echo
clear_color
}
set_color() {
local color="$1"
local weight="$2"
printf "\x1B[%d;%dm" $(( 30 + $color )) "$( [ "$weight" = "bold" ] && echo 1 || echo 22 )"
}
clear_color() {
printf "\x1B[0m"
}
plural() {
[ "$1" -eq 1 ] || echo "s"
}
_buffer=""
buffer() {
_buffer="${_buffer}$("$@")"
}
flush() {
printf "%s" "$_buffer"
_buffer=""
}
finish() {
flush
printf "\n"
}
trap finish EXIT
while IFS= read -r line; do
case "$line" in
"begin "* )
let index+=1
name="${line#* $index }"
buffer begin
flush
;;
"ok "* )
skip_expr="ok $index # skip (\(([^)]*)\))?"
if [[ "$line" =~ $skip_expr ]]; then
let skipped+=1
buffer skip "${BASH_REMATCH[2]}"
else
buffer pass
fi
;;
"not ok "* )
let failures+=1
buffer fail
;;
"# "* )
buffer log "${line:2}"
;;
esac
done
buffer summary

52
test/bats/bats-preprocess Executable file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -e
encode_name() {
local name="$1"
local result="test_"
if [[ ! "$name" =~ [^[:alnum:]\ _-] ]]; then
name="${name//_/-5f}"
name="${name//-/-2d}"
name="${name// /_}"
result+="$name"
else
local length="${#name}"
local char i
for ((i=0; i<length; i++)); do
char="${name:$i:1}"
if [ "$char" = " " ]; then
result+="_"
elif [[ "$char" =~ [[:alnum:]] ]]; then
result+="$char"
else
result+="$(printf -- "-%02x" \'"$char")"
fi
done
fi
echo "$result"
}
tests=()
index=0
pattern='^ *@test *([^ ].*) *\{ *(.*)$'
while IFS= read -r line; do
let index+=1
if [[ "$line" =~ $pattern ]]; then
quoted_name="${BASH_REMATCH[1]}"
body="${BASH_REMATCH[2]}"
name="$(eval echo "$quoted_name")"
encoded_name="$(encode_name "$name")"
tests["${#tests[@]}"]="$encoded_name"
echo "${encoded_name}() { bats_test_begin ${quoted_name} ${index}; ${body}"
else
printf "%s\n" "$line"
fi
done
for test_name in "${tests[@]}"; do
echo "bats_test_function ${test_name}"
done

View file

@ -1,116 +0,0 @@
#!/bin/bash
# Set up test framework
source assert.sh
# Testing that services are running and pop3 is disabled
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/lib/postfix/master'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/sbin/saslauthd'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/sbin/amavisd-new'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/sbin/opendkim'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/sbin/opendmarc'" 0
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/lib/courier/courier/courierpop3d'" 1
assert_raises "docker exec mail ps aux --forest | grep -v grep | grep '/usr/bin/python /usr/bin/fail2ban-server'" 0
# Testing services of pop3 container
assert_raises "docker exec mail_pop3 ps aux --forest | grep -v grep | grep '/usr/lib/courier/courier/courierpop3d'" 0
# Testing IMAP server
assert_raises "docker exec mail nc -w 1 0.0.0.0 143 | grep '* OK' | grep 'STARTTLS' | grep 'Courier-IMAP ready'" 0
assert_raises "docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 143 < /tmp/test/auth/imap-auth.txt'" 0
# Testing POP3 server on pop3 container
assert_raises "docker exec mail_pop3 nc -w 1 0.0.0.0 110 | grep '+OK'" 0
assert_raises "docker exec mail_pop3 /bin/sh -c 'nc -w 1 0.0.0.0 110 < /tmp/test/auth/pop3-auth.txt'" 0
# Testing SASL
assert_raises "docker exec mail testsaslauthd -u user2 -r otherdomain.tld -p mypassword | grep 'OK \"Success.\"'" 0
assert_raises "docker exec mail testsaslauthd -u user2 -r otherdomain.tld -p BADPASSWORD | grep 'NO \"authentication failed\"'" 0
assert_raises "docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-plain.txt' | grep 'Authentication successful'"
assert_raises "docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login.txt' | grep 'Authentication successful'"
# Testing user creation
assert "docker exec mail sasldblistusers2" "user1@localhost.localdomain: userPassword\nuser2@otherdomain.tld: userPassword"
assert "docker exec mail ls -A /var/mail/localhost.localdomain/user1" ".Drafts\n.Sent\n.Trash\ncourierimapsubscribed\ncur\nnew\ntmp"
assert "docker exec mail ls -A /var/mail/otherdomain.tld/user2" ".Drafts\n.Sent\n.Trash\ncourierimapsubscribed\ncur\nnew\ntmp"
# Testing `vhost` creation
assert "docker exec mail cat /etc/postfix/vhost" "localhost.localdomain\notherdomain.tld"
# Testing that mail is received for existing user
assert_raises "docker exec mail grep 'status=sent (delivered to maildir)' /var/log/mail.log" 0
assert "docker exec mail ls -A /var/mail/localhost.localdomain/user1/new | wc -l" "2"
# Testing that mail is rejected for non existing user
assert_raises "docker exec mail grep '<nouser@localhost.localdomain>: Recipient address rejected: User unknown in virtual mailbox table' /var/log/mail.log" 0
# Testing that mail is received for existing alias
assert_raises "docker exec mail grep 'to=<user1@localhost.localdomain>, orig_to=<alias1@localhost.localdomain>' /var/log/mail.log | grep 'status=sent'" 0
# Testing that mail is redirected for external alias
assert_raises "docker exec mail grep -- '-> <external1@otherdomain.tld>' /var/log/mail.log" 0
# Testing that a SPAM is rejected
assert_raises "docker exec mail grep 'Blocked SPAM' /var/log/mail.log | grep spam@external.tld" 0
# Testing that a Virus is rejected
assert_raises "docker exec mail grep 'Blocked INFECTED' /var/log/mail.log | grep virus@external.tld" 0
# Testing presence of freshclam CRON
assert "docker exec mail crontab -l" "0 1 * * * /usr/bin/freshclam --quiet"
# Testing that log don't display errors
assert_raises "docker exec mail grep 'non-null host address bits in' /var/log/mail.log" 1
assert_raises "docker exec mail grep ': error:' /var/log/mail.log" 1
# Testing that pop3 container log don't display errors
assert_raises "docker exec mail_pop3 grep 'non-null host address bits in' /var/log/mail.log" 1
assert_raises "docker exec mail_pop3 grep ': error:' /var/log/mail.log" 1
# Testing Spamssassin config in Amavis
assert_raises "docker exec mail_pop3 grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" 0
assert_raises "docker exec mail_pop3 grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 6.31'" 0
assert_raises "docker exec mail_pop3 grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 6.31'" 0
assert_raises "docker exec mail grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 1.0'" 0
assert_raises "docker exec mail grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'" 0
assert_raises "docker exec mail grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'" 0
# Testing OpenDKIM
assert "docker exec mail cat /etc/opendkim/KeyTable | wc -l" "2"
assert "docker exec mail ls -l /etc/opendkim/keys/ | grep '^d' | wc -l" "2"
# Testing OpenDMARC
assert "docker exec mail cat /etc/opendmarc.conf | grep ^AuthservID | wc -l" "1"
assert "docker exec mail cat /etc/opendmarc.conf | grep ^TrustedAuthservID | wc -l" "1"
# Testing hostname config
assert "docker exec mail cat /etc/mailname" "my-domain.com"
# Testing presence of LetsEncrypt signed certs
assert_raises "docker exec mail grep 'BEGIN CERTIFICATE' /etc/ssl/certs/lets-encrypt-x1-cross-signed.pem" "0"
assert_raises "docker exec mail grep 'BEGIN CERTIFICATE' /etc/ssl/certs/lets-encrypt-x2-cross-signed.pem" "0"
# Testing generated ssl certs
assert_raises "docker exec mail openssl s_client -connect 0.0.0.0:587 -starttls smtp -CApath /etc/ssl/certs/ | grep 'Verify return code: 0 (ok)'" "0"
# Testing fail2ban
assert_raises "docker exec mail fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'" 1
docker exec mail fail2ban-client set sasl delignoreip 127.0.0.1/8 &> /dev/null
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt' &> /dev/null
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt' &> /dev/null
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt' &> /dev/null
sleep 10
assert_raises "docker exec mail fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'" 0
docker exec mail fail2ban-client set sasl addignoreip 127.0.0.1/8 &> /dev/null
docker exec mail fail2ban-client set sasl unbanip 127.0.0.1 &> /dev/null
sleep 10
assert_raises "docker exec mail fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'" 1
# Ending tests
assert_end

340
test/tests.bats Normal file
View file

@ -0,0 +1,340 @@
#
# processes
#
@test "checking process: postfix" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/lib/postfix/master'"
[ "$status" -eq 0 ]
}
@test "checking process: saslauthd" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/saslauthd'"
[ "$status" -eq 0 ]
}
@test "checking process: clamd" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/clamd'"
[ "$status" -eq 0 ]
}
@test "checking process: new" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/amavisd-new'"
[ "$status" -eq 0 ]
}
@test "checking process: opendkim" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/opendkim'"
[ "$status" -eq 0 ]
}
@test "checking process: opendmarc" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/sbin/opendmarc'"
[ "$status" -eq 0 ]
}
@test "checking process: fail2ban" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/bin/python /usr/bin/fail2ban-server'"
[ "$status" -eq 0 ]
}
@test "checking process: courierpop3d (disabled in default configuration)" {
run docker exec mail /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/lib/courier/courier/courierpop3d'"
[ "$status" -eq 1 ]
}
@test "checking process: courierpop3d (pop3 server enabled)" {
run docker exec mail_pop3 /bin/bash -c "ps aux --forest | grep -v grep | grep '/usr/lib/courier/courier/courierpop3d'"
[ "$status" -eq 0 ]
}
#
# imap
#
@test "checking imap: server is ready with STARTTLS" {
run docker exec mail /bin/bash -c "nc -w 1 0.0.0.0 143 | grep '* OK' | grep 'STARTTLS' | grep 'Courier-IMAP ready'"
[ "$status" -eq 0 ]
}
@test "checking imap: authentication works" {
run docker exec mail /bin/sh -c "nc -w 1 0.0.0.0 143 < /tmp/test/auth/imap-auth.txt"
[ "$status" -eq 0 ]
}
#
# pop
#
@test "checking pop: server is ready" {
run docker exec mail_pop3 /bin/bash -c "nc -w 1 0.0.0.0 110 | grep '+OK'"
[ "$status" -eq 0 ]
}
@test "checking pop: authentication works" {
run docker exec mail_pop3 /bin/sh -c "nc -w 1 0.0.0.0 110 < /tmp/test/auth/pop3-auth.txt"
[ "$status" -eq 0 ]
}
#
# sasl
#
@test "checking sasl: testsaslauthd works with good password" {
run docker exec mail /bin/sh -c "testsaslauthd -u user2 -r otherdomain.tld -p mypassword | grep 'OK \"Success.\"'"
[ "$status" -eq 0 ]
}
@test "checking sasl: testsaslauthd fails with bad password" {
run docker exec mail /bin/sh -c "testsaslauthd -u user2 -r otherdomain.tld -p BADPASSWORD | grep 'NO \"authentication failed\"'"
[ "$status" -eq 0 ]
}
#
# smtp
#
@test "checking smtp: authentication works with good password (plain)" {
run docker exec mail /bin/sh -c "nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-plain.txt | grep 'Authentication successful'"
[ "$status" -eq 0 ]
}
@test "checking smtp: authentication fails with wrong password (plain)" {
run docker exec mail /bin/sh -c "nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-plain-wrong.txt | grep 'authentication failed'"
[ "$status" -eq 0 ]
}
@test "checking smtp: authentication works with good password (login)" {
run docker exec mail /bin/sh -c "nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login.txt | grep 'Authentication successful'"
[ "$status" -eq 0 ]
}
@test "checking smtp: authentication fails with wrong password (login)" {
run docker exec mail /bin/sh -c "nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt | grep 'authentication failed'"
[ "$status" -eq 0 ]
}
@test "checking smtp: delivers mail to existing account" {
run docker exec mail /bin/sh -c "grep 'status=sent (delivered to maildir)' /var/log/mail.log | wc -l"
[ "$status" -eq 0 ]
[ "$output" -eq 2 ]
}
@test "checking smtp: delivers mail to existing alias" {
run docker exec mail /bin/sh -c "grep 'to=<user1@localhost.localdomain>, orig_to=<alias1@localhost.localdomain>' /var/log/mail.log | grep 'status=sent' | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 1 ]
}
@test "checking smtp: user1 should have received 2 mails" {
run docker exec mail /bin/sh -c "ls -A /var/mail/localhost.localdomain/user1/new | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 2 ]
}
@test "checking smtp: rejects mail to unknown user" {
run docker exec mail /bin/sh -c "grep '<nouser@localhost.localdomain>: Recipient address rejected: User unknown in virtual mailbox table' /var/log/mail.log | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 1 ]
}
@test "checking smtp: redirects mail to external alias" {
run docker exec mail /bin/sh -c "grep -- '-> <external1@otherdomain.tld>' /var/log/mail.log | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 1 ]
}
@test "checking smtp: rejects spam" {
run docker exec mail /bin/sh -c "grep 'Blocked SPAM' /var/log/mail.log | grep spam@external.tld | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 1 ]
}
@test "checking smtp: rejects virus" {
run docker exec mail /bin/sh -c "grep 'Blocked INFECTED' /var/log/mail.log | grep virus@external.tld | wc -l"
[ "$status" -eq 0 ]
[ "$output" = 1 ]
}
#
# accounts
#
@test "checking accounts: user accounts" {
run docker exec mail sasldblistusers2
[ "$status" -eq 0 ]
[ "${lines[0]}" = "user1@localhost.localdomain: userPassword" ]
[ "${lines[1]}" = "user2@otherdomain.tld: userPassword" ]
}
@test "checking accounts: user mail folders for user1" {
run docker exec mail ls -A /var/mail/localhost.localdomain/user1
[ "$status" -eq 0 ]
[ "${lines[0]}" = ".Drafts" ]
[ "${lines[1]}" = ".Sent" ]
[ "${lines[2]}" = ".Trash" ]
[ "${lines[3]}" = "courierimapsubscribed" ]
[ "${lines[4]}" = "cur" ]
[ "${lines[5]}" = "new" ]
[ "${lines[6]}" = "tmp" ]
}
@test "checking accounts: user mail folders for user2" {
run docker exec mail ls -A /var/mail/otherdomain.tld/user2
[ "$status" -eq 0 ]
[ "${lines[0]}" = ".Drafts" ]
[ "${lines[1]}" = ".Sent" ]
[ "${lines[2]}" = ".Trash" ]
[ "${lines[3]}" = "courierimapsubscribed" ]
[ "${lines[4]}" = "cur" ]
[ "${lines[5]}" = "new" ]
[ "${lines[6]}" = "tmp" ]
}
#
# postfix
#
@test "checking postfix: vhost file is correct" {
run docker exec mail cat /etc/postfix/vhost
[ "$status" -eq 0 ]
[ "${lines[0]}" = "localhost.localdomain" ]
[ "${lines[1]}" = "otherdomain.tld" ]
}
#
# spamassassin
#
@test "checking spamassassin: docker env variables are set correctly (default)" {
run docker exec mail_pop3 /bin/sh -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'"
[ "$status" -eq 0 ]
run docker exec mail_pop3 /bin/sh -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 6.31'"
[ "$status" -eq 0 ]
run docker exec mail_pop3 /bin/sh -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 6.31'"
[ "$status" -eq 0 ]
}
@test "checking spamassassin: docker env variables are set correctly (custom)" {
run docker exec mail /bin/sh -c "grep '\$sa_tag_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 1.0'"
[ "$status" -eq 0 ]
run docker exec mail /bin/sh -c "grep '\$sa_tag2_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 2.0'"
[ "$status" -eq 0 ]
run docker exec mail /bin/sh -c "grep '\$sa_kill_level_deflt' /etc/amavis/conf.d/20-debian_defaults | grep '= 3.0'"
[ "$status" -eq 0 ]
}
#
# opendkim
#
@test "checking opendkim: /etc/opendkim/KeyTable should contain 2 entries" {
run docker exec mail /bin/sh -c "cat /etc/opendkim/KeyTable | wc -l"
[ "$status" -eq 0 ]
[ "$output" -eq 2 ]
}
@test "checking opendkim: /etc/opendkim/keys/ should contain 2 entries" {
run docker exec mail /bin/sh -c "ls -l /etc/opendkim/keys/ | grep '^d' | wc -l"
[ "$status" -eq 0 ]
[ "$output" -eq 2 ]
}
#
# opendmarc
#
@test "checking opendkim: server fqdn should be added to /etc/opendmarc.conf as AuthservID" {
run docker exec mail grep ^AuthservID /etc/opendmarc.conf
[ "$status" -eq 0 ]
[ "$output" = "AuthservID mail.my-domain.com" ]
}
@test "checking opendkim: server fqdn should be added to /etc/opendmarc.conf as TrustedAuthservIDs" {
run docker exec mail grep ^TrustedAuthservID /etc/opendmarc.conf
[ "$status" -eq 0 ]
[ "$output" = "TrustedAuthservIDs mail.my-domain.com" ]
}
#
# letsencrypt
#
@test "checking letsencrypt: lets-encrypt-x1-cross-signed.pem is installed" {
run docker exec mail grep 'BEGIN CERTIFICATE' /etc/ssl/certs/lets-encrypt-x1-cross-signed.pem
[ "$status" -eq 0 ]
}
@test "checking letsencrypt: lets-encrypt-x2-cross-signed.pem is installed" {
run docker exec mail grep 'BEGIN CERTIFICATE' /etc/ssl/certs/lets-encrypt-x2-cross-signed.pem
[ "$status" -eq 0 ]
}
#
# ssl
#
@test "checking ssl: generated default cert is installed" {
run docker exec mail /bin/sh -c "openssl s_client -connect 0.0.0.0:587 -starttls smtp -CApath /etc/ssl/certs/ | grep 'Verify return code: 0 (ok)'"
[ "$status" -eq 0 ]
}
#
# fail2ban
#
@test "checking fail2ban: localhost is not banned" {
run docker exec mail /bin/sh -c "fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'"
[ "$status" -eq 1 ]
}
@test "checking fail2ban: ban ip on multiple failed login" {
docker exec mail fail2ban-client status sasl
docker exec mail fail2ban-client set sasl delignoreip 127.0.0.1/8
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt'
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt'
docker exec mail /bin/sh -c 'nc -w 1 0.0.0.0 25 < /tmp/test/auth/smtp-auth-login-wrong.txt'
sleep 5
run docker exec mail /bin/sh -c "fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'"
[ "$status" -eq 0 ]
}
@test "checking fail2ban: unban ip works" {
docker exec mail fail2ban-client set sasl addignoreip 127.0.0.1/8
docker exec mail fail2ban-client set sasl unbanip 127.0.0.1
sleep 5
run docker exec mail /bin/sh -c "fail2ban-client status sasl | grep 'IP list:\s*127.0.0.1'"
[ "$status" -eq 1 ]
}
#
# system
#
@test "checking system: freshclam cron is enabled" {
run docker exec mail crontab -l
[ "$status" -eq 0 ]
[ "$output" = "0 1 * * * /usr/bin/freshclam --quiet" ]
}
@test "checking system: /var/log/mail.log is error free" {
run docker exec mail grep 'non-null host address bits in' /var/log/mail.log
[ "$status" -eq 1 ]
run docker exec mail grep ': error:' /var/log/mail.log
[ "$status" -eq 1 ]
run docker exec mail_pop3 grep 'non-null host address bits in' /var/log/mail.log
[ "$status" -eq 1 ]
run docker exec mail_pop3 grep ': error:' /var/log/mail.log
[ "$status" -eq 1 ]
}
@test "checking system: sets the server fqdn" {
run docker exec mail hostname
[ "$status" -eq 0 ]
[ "$output" = "mail.my-domain.com" ]
}
@test "checking system: sets the server domain name in /etc/mailname" {
run docker exec mail cat /etc/mailname
[ "$status" -eq 0 ]
[ "$output" = "my-domain.com" ]
}