From 227cde12ba6189859cd274de09a7b344ce870863 Mon Sep 17 00:00:00 2001 From: Alexander Larsson Date: Wed, 9 Oct 2019 12:06:22 +0200 Subject: [PATCH] tests: Redo httpd server setup Rather than this weird shell loop with a timeout and polling we move the httpd spawning entirely into the python code, and use a pipe to synchronize the spawning. This way we can also use the shell job control to properly clean up any running processes from the test suite. Additionally, this adds some (lame) support for token handling in the test webserver, where you for any file foo can create a foo.need_token containing a token that is needed for that file. --- tests/http-utils-test-server.py | 37 +++++++++++------- tests/libtest.sh | 19 +++++++--- tests/oci-registry-server.py | 33 ++++++++++------ tests/test-http-utils.sh | 6 +-- tests/test-oci-registry.sh | 6 +-- tests/test-repo.sh | 20 +++++----- tests/test-unsigned-summaries.sh | 4 +- tests/test-webserver.sh | 33 ++-------------- tests/web-server.py | 64 ++++++++++++++++++++++++++++++++ 9 files changed, 142 insertions(+), 80 deletions(-) mode change 100644 => 100755 tests/http-utils-test-server.py mode change 100644 => 100755 tests/oci-registry-server.py create mode 100755 tests/web-server.py diff --git a/tests/http-utils-test-server.py b/tests/http-utils-test-server.py old mode 100644 new mode 100755 index 01beada2..698d7297 --- a/tests/http-utils-test-server.py +++ b/tests/http-utils-test-server.py @@ -7,15 +7,11 @@ import gzip import sys import time import zlib +import os -if sys.version_info[0] >= 3: - from urllib.parse import parse_qs - import http.server as http_server - from io import BytesIO -else: - from urllib.parse import parse_qs - import http.server as http_server - from io import StringIO as BytesIO +from urllib.parse import parse_qs +import http.server as http_server +from io import BytesIO server_start_time = int(time.time()) @@ -93,11 +89,24 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): else: self.wfile.write(contents.encode('utf-8')) -def test(): - if sys.version_info[0] >= 3: - http_server.test(RequestHandler, port=0) - else: - http_server.test(RequestHandler) +def run(dir): + RequestHandler.protocol_version = "HTTP/1.0" + httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler) + host, port = httpd.socket.getsockname()[:2] + with open("httpd-port", 'w') as file: + file.write("%d" % port) + try: + os.write(3, bytes("Started\n", 'utf-8')); + except: + pass + print("Serving HTTP on port %d" % port); + if dir: + os.chdir(dir) + httpd.serve_forever() if __name__ == '__main__': - test() + dir = None + if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: + dir = sys.argv[1] + + run(dir) diff --git a/tests/libtest.sh b/tests/libtest.sh index 230ec23d..61bc3038 100644 --- a/tests/libtest.sh +++ b/tests/libtest.sh @@ -241,6 +241,16 @@ make_runtime () { flatpak build-commit-from --disable-fsync --src-repo=${RUNTIME_REPO} --force ${GPGARGS} repos/${REPONAME} ${RUNTIME_REF} } +httpd () { + COMMAND=${1:-web-server.py} + DIR=${2:-repos} + + rm -f httpd-pipe + mkfifo httpd-pipe + $(dirname $0)/$COMMAND "$DIR" 3> httpd-pipe & + read < httpd-pipe +} + setup_repo_no_add () { REPONAME=${1:-test} if [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then @@ -254,9 +264,7 @@ setup_repo_no_add () { GPGARGS="${GPGARGS:-${FL_GPGARGS}}" $(dirname $0)/make-test-app.sh repos/${REPONAME} "" "${BRANCH}" "${COLLECTION_ID}" > /dev/null update_repo $REPONAME "${COLLECTION_ID}" if [ $REPONAME == "test" ]; then - $(dirname $0)/test-webserver.sh repos - FLATPAK_HTTP_PID=$(cat httpd-pid) - mv httpd-port httpd-port-main + httpd fi } @@ -266,7 +274,7 @@ setup_repo () { setup_repo_no_add "$@" - port=$(cat httpd-port-main) + port=$(cat httpd-port) if [ x${GPGPUBKEY:-${FL_GPG_HOMEDIR}/pubring.gpg} != x ]; then import_args=--gpg-import=${GPGPUBKEY:-${FL_GPG_HOMEDIR}/pubring.gpg} else @@ -437,9 +445,10 @@ if ! /bin/kill -0 "$DBUS_SESSION_BUS_PID"; then fi cleanup () { - /bin/kill -9 $DBUS_SESSION_BUS_PID ${FLATPAK_HTTP_PID:-} + /bin/kill -9 $DBUS_SESSION_BUS_PID gpg-connect-agent --homedir "${FL_GPG_HOMEDIR}" killagent /bye || true fusermount -u $XDG_RUNTIME_DIR/doc || : + kill $(jobs -p) &> /dev/null || true if test -n "${TEST_SKIP_CLEANUP:-}"; then echo "Skipping cleanup of ${TEST_DATA_DIR}" else diff --git a/tests/oci-registry-server.py b/tests/oci-registry-server.py old mode 100644 new mode 100755 index 83fd4db2..88024942 --- a/tests/oci-registry-server.py +++ b/tests/oci-registry-server.py @@ -7,12 +7,8 @@ import os import sys import time -if sys.version_info[0] >= 3: - from urllib.parse import parse_qs - import http.server as http_server -else: - from urllib.parse import parse_qs - import http.server as http_server +from urllib.parse import parse_qs +import http.server as http_server repositories = {} icons = {} @@ -246,11 +242,24 @@ class RequestHandler(http_server.BaseHTTPRequestHandler): self.end_headers() return -def test(): - if sys.version_info[0] >= 3: - http_server.test(RequestHandler, port=0) - else: - http_server.test(RequestHandler) +def run(dir): + RequestHandler.protocol_version = "HTTP/1.0" + httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler) + host, port = httpd.socket.getsockname()[:2] + with open("httpd-port", 'w') as file: + file.write("%d" % port) + try: + os.write(3, bytes("Started\n", 'utf-8')); + except: + pass + print("Serving HTTP on port %d" % port); + if dir: + os.chdir(dir) + httpd.serve_forever() if __name__ == '__main__': - test() + dir = None + if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: + dir = sys.argv[1] + + run(dir) diff --git a/tests/test-http-utils.sh b/tests/test-http-utils.sh index 2247d2b3..8ff3e169 100755 --- a/tests/test-http-utils.sh +++ b/tests/test-http-utils.sh @@ -21,10 +21,8 @@ set -euo pipefail . $(dirname $0)/libtest.sh -$(dirname $0)/test-webserver.sh "" "python3 $test_srcdir/http-utils-test-server.py 0" -FLATPAK_HTTP_PID=$(cat httpd-pid) -mv httpd-port httpd-port-main -port=$(cat httpd-port-main) +httpd http-utils-test-server.py . +port=$(cat httpd-port) assert_result() { test_string=$1 diff --git a/tests/test-oci-registry.sh b/tests/test-oci-registry.sh index 9f214819..a49f932a 100755 --- a/tests/test-oci-registry.sh +++ b/tests/test-oci-registry.sh @@ -35,10 +35,8 @@ fi # Start the fake registry server -$(dirname $0)/test-webserver.sh "" "python3 $test_srcdir/oci-registry-server.py 0" -FLATPAK_HTTP_PID=$(cat httpd-pid) -mv httpd-port httpd-port-main -port=$(cat httpd-port-main) +httpd oci-registry-server.py . +port=$(cat httpd-port) client="python3 $test_srcdir/oci-registry-client.py 127.0.0.1:$port" setup_repo_no_add oci diff --git a/tests/test-repo.sh b/tests/test-repo.sh index c9f141ed..96535ffe 100644 --- a/tests/test-repo.sh +++ b/tests/test-repo.sh @@ -50,7 +50,7 @@ if [ x${USE_COLLECTIONS_IN_CLIENT-} == xyes ] ; then elif [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then # Set a collection ID and GPG on the server, but not in the client configuration setup_repo_no_add test-no-gpg org.test.Collection.NoGpg - port=$(cat httpd-port-main) + port=$(cat httpd-port) flatpak remote-add ${U} --no-gpg-verify test-no-gpg-repo "http://127.0.0.1:${port}/test-no-gpg" else GPGPUBKEY="" GPGARGS="" setup_repo test-no-gpg @@ -64,13 +64,13 @@ GPGPUBKEY="${FL_GPG_HOMEDIR2}/pubring.gpg" GPGARGS="${FL_GPGARGS2}" setup_repo t #remote with missing GPG key # Don’t use --collection-id= here, or the collections code will grab the appropriate # GPG key from one of the previously-configured remotes with the same collection ID. -port=$(cat httpd-port-main) +port=$(cat httpd-port) if flatpak remote-add ${U} test-missing-gpg-repo "http://127.0.0.1:${port}/test"; then assert_not_reached "Should fail metadata-update due to missing gpg key" fi #remote with wrong GPG key -port=$(cat httpd-port-main) +port=$(cat httpd-port) if flatpak remote-add ${U} --gpg-import=${FL_GPG_HOMEDIR2}/pubring.gpg test-wrong-gpg-repo "http://127.0.0.1:${port}/test"; then assert_not_reached "Should fail metadata-update due to wrong gpg key" fi @@ -170,7 +170,7 @@ fi echo "ok missing remote name auto-corrects for install" -port=$(cat httpd-port-main) +port=$(cat httpd-port) if ${FLATPAK} ${U} install -y http://127.0.0.1:${port}/nonexistent.flatpakref 2> install-error-log; then assert_not_reached "Should not be able to install a nonexistent flatpakref" fi @@ -184,7 +184,7 @@ setup_repo_no_add flatpakref org.test.Collection.Flatpakref cat << EOF > repos/flatpakref/flatpakref-repo.flatpakrepo [Flatpak Repo] Version=1 -Url=http://127.0.0.1:$(cat httpd-port-main)/flatpakref/ +Url=http://127.0.0.1:$(cat httpd-port)/flatpakref/ Title=The Title GPGKey=${FL_GPG_BASE64} EOF @@ -197,9 +197,9 @@ cat << EOF > org.test.Hello.flatpakref [Flatpak Ref] Name=org.test.Hello Branch=master -Url=http://127.0.0.1:$(cat httpd-port-main)/flatpakref +Url=http://127.0.0.1:$(cat httpd-port)/flatpakref GPGKey=${FL_GPG_BASE64} -RuntimeRepo=http://127.0.0.1:$(cat httpd-port-main)/flatpakref/flatpakref-repo.flatpakrepo +RuntimeRepo=http://127.0.0.1:$(cat httpd-port)/flatpakref/flatpakref-repo.flatpakrepo EOF ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello @@ -427,7 +427,7 @@ echo "ok eol-rebase" ${FLATPAK} ${U} install -y test-repo org.test.Platform -port=$(cat httpd-port-main) +port=$(cat httpd-port) UPDATE_REPO_ARGS="--redirect-url=http://127.0.0.1:${port}/test-gpg3 --gpg-import=${FL_GPG_HOMEDIR2}/pubring.gpg" update_repo GPGPUBKEY="${FL_GPG_HOMEDIR2}/pubring.gpg" GPGARGS="${FL_GPGARGS2}" setup_repo_no_add test-gpg3 org.test.Collection.test master @@ -505,7 +505,7 @@ assert_not_file_has_content list-log "org\.test\.Hello" assert_not_file_has_content list-log "org\.test\.Platform" # Disable the remote to make sure we don't do i/o -port=$(cat httpd-port-main) +port=$(cat httpd-port) ${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/disable-test" test-repo ${FLATPAK} ${U} install -y --no-pull test-repo org.test.Hello @@ -528,7 +528,7 @@ assert_not_file_has_content list-log "org\.test\.Hello" assert_not_file_has_content list-log "org\.test\.Platform" # Disable the remote to make sure we don't do i/o -port=$(cat httpd-port-main) +port=$(cat httpd-port) ${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/disable-test" test-repo # Note: The partial ref is only auto-corrected without user interaction because we're using -y diff --git a/tests/test-unsigned-summaries.sh b/tests/test-unsigned-summaries.sh index 9e6ed537..0701473d 100755 --- a/tests/test-unsigned-summaries.sh +++ b/tests/test-unsigned-summaries.sh @@ -101,10 +101,10 @@ cat << EOF > org.test.App.flatpakref Title=Test App Name=org.test.App Branch=master -Url=http://127.0.0.1:$(cat httpd-port-main)/test +Url=http://127.0.0.1:$(cat httpd-port)/test IsRuntime=False GPGKey=${FL_GPG_BASE64} -#RuntimeRepo=http://127.0.0.1:$(cat httpd-port-main)/test +#RuntimeRepo=http://127.0.0.1:$(cat httpd-port)/test DeployCollectionID=org.test.Collection EOF diff --git a/tests/test-webserver.sh b/tests/test-webserver.sh index a8e2bc17..1e5fbb75 100755 --- a/tests/test-webserver.sh +++ b/tests/test-webserver.sh @@ -3,33 +3,8 @@ set -euo pipefail dir=$1 -cmd=${2:-python3 -m http.server 0} -test_tmpdir=$(pwd) -[ "$dir" != "" ] && cd ${dir} -echo "Running web server: PYTHONUNBUFFERED=1 setsid $cmd" >&2 -touch ${test_tmpdir}/httpd-output -env PYTHONUNBUFFERED=1 setsid $cmd >${test_tmpdir}/httpd-output & -child_pid=$! -echo "Web server pid: $child_pid" >&2 - -for x in $(seq 300); do - echo "Waiting for web server ($x/300)..." >&2 - # Snapshot the output - cp ${test_tmpdir}/httpd-output{,.tmp} - sed -ne 's/^/# httpd-output.tmp: /' < ${test_tmpdir}/httpd-output.tmp >&2 - echo >&2 - # If it's non-empty, see whether it matches our regexp - if test -s ${test_tmpdir}/httpd-output.tmp; then - sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) (http://0.0.0.0:[0-9]*/) \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port - if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then - # If so, we've successfully extracted the port - break - fi - fi - sleep 0.1 -done -port=$(cat ${test_tmpdir}/httpd-port) -echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address -echo "$child_pid" > ${test_tmpdir}/httpd-pid -echo "Started web server '$cmd': process $child_pid on port $port" >&2 +rm -f httpd-pipe +mkfifo httpd-pipe +$(dirname $0)/web-server.py "$dir" 3> httpd-pipe & +read < httpd-pipe diff --git a/tests/web-server.py b/tests/web-server.py new file mode 100755 index 00000000..ce79d93d --- /dev/null +++ b/tests/web-server.py @@ -0,0 +1,64 @@ +#!/usr/bin/python3 + +from wsgiref.handlers import format_date_time +from email.utils import parsedate +from calendar import timegm +import gzip +import sys +import time +import zlib +import os +from http import HTTPStatus +from urllib.parse import parse_qs +import http.server as http_server +from io import BytesIO +import sys + +class RequestHandler(http_server.SimpleHTTPRequestHandler): + def handle_tokens(self): + need_token_path = self.translate_path(self.path) + ".need_token" + if os.path.isfile(need_token_path): + with open(need_token_path, 'r') as content_file: + token_content = content_file.read() + token = None + auth = self.headers.get("Authorization") + if auth and auth.startswith("Bearer "): + token = auth[7:] + if token == None: + self.send_response(HTTPStatus.UNAUTHORIZED, "No token") + self.end_headers() + return True + if token != token_content: + self.send_response(HTTPStatus.UNAUTHORIZED, "Wrong token") + self.end_headers() + return True + return False + + def do_GET(self): + if self.handle_tokens(): + return None + return super().do_GET() + +def run(dir): + RequestHandler.protocol_version = "HTTP/1.0" + httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler) + host, port = httpd.socket.getsockname()[:2] + with open("httpd-port", 'w') as file: + file.write("%d" % port) + with open("httpd-pid", 'w') as file: + file.write("%d" % os.getpid()) + try: + os.write(3, bytes("Started\n", 'utf-8')); + except: + pass + print("Serving HTTP on port %d" % port); + if dir: + os.chdir(dir) + httpd.serve_forever() + +if __name__ == '__main__': + dir = None + if len(sys.argv) >= 2 and len(sys.argv[1]) > 0: + dir = sys.argv[1] + + run(dir)