tests: Redo httpd server setup

Rather than this weird shell loop with a timeout and polling we
move the httpd spawning entirely into the python code, and use
a pipe to synchronize the spawning. This way we can also use
the shell job control to properly clean up any running processes
from the test suite.

Additionally, this adds some (lame) support for token handling in the
test webserver, where you for any file foo can create a foo.need_token
containing a token that is needed for that file.

tests/http-utils-test-server.py

@@ -7,15 +7,11 @@ import gzip
 import sys
 import time
 import zlib
+import os
 
-if sys.version_info[0] >= 3:
-    from urllib.parse import parse_qs
-    import http.server as http_server
-    from io import BytesIO
-else:
-    from urllib.parse import parse_qs
-    import http.server as http_server
-    from io import StringIO as BytesIO
+from urllib.parse import parse_qs
+import http.server as http_server
+from io import BytesIO
 
 server_start_time = int(time.time())
 
@@ -93,11 +89,24 @@ class RequestHandler(http_server.BaseHTTPRequestHandler):
             else:
                 self.wfile.write(contents.encode('utf-8'))
 
-def test():
-    if sys.version_info[0] >= 3:
-        http_server.test(RequestHandler, port=0)
-    else:
-        http_server.test(RequestHandler)
+def run(dir):
+    RequestHandler.protocol_version = "HTTP/1.0"
+    httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler)
+    host, port = httpd.socket.getsockname()[:2]
+    with open("httpd-port", 'w') as file:
+        file.write("%d" % port)
+    try:
+        os.write(3, bytes("Started\n", 'utf-8'));
+    except:
+        pass
+    print("Serving HTTP on port %d" % port);
+    if dir:
+        os.chdir(dir)
+    httpd.serve_forever()
 
 if __name__ == '__main__':
-    test()
+    dir = None
+    if len(sys.argv) >= 2 and len(sys.argv[1]) > 0:
+        dir = sys.argv[1]
+
+    run(dir)

tests/libtest.sh

@@ -241,6 +241,16 @@ make_runtime () {
     flatpak build-commit-from --disable-fsync --src-repo=${RUNTIME_REPO} --force ${GPGARGS} repos/${REPONAME}  ${RUNTIME_REF}
 }
 
+httpd () {
+    COMMAND=${1:-web-server.py}
+    DIR=${2:-repos}
+
+    rm -f httpd-pipe
+    mkfifo httpd-pipe
+    $(dirname $0)/$COMMAND "$DIR" 3> httpd-pipe &
+    read < httpd-pipe
+}
+
 setup_repo_no_add () {
     REPONAME=${1:-test}
     if [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then
@@ -254,9 +264,7 @@ setup_repo_no_add () {
     GPGARGS="${GPGARGS:-${FL_GPGARGS}}" $(dirname $0)/make-test-app.sh repos/${REPONAME} "" "${BRANCH}" "${COLLECTION_ID}" > /dev/null
     update_repo $REPONAME "${COLLECTION_ID}"
     if [ $REPONAME == "test" ]; then
-        $(dirname $0)/test-webserver.sh repos
-        FLATPAK_HTTP_PID=$(cat httpd-pid)
-        mv httpd-port httpd-port-main
+        httpd
     fi
 }
 
@@ -266,7 +274,7 @@ setup_repo () {
 
     setup_repo_no_add "$@"
 
-    port=$(cat httpd-port-main)
+    port=$(cat httpd-port)
     if [ x${GPGPUBKEY:-${FL_GPG_HOMEDIR}/pubring.gpg} != x ]; then
         import_args=--gpg-import=${GPGPUBKEY:-${FL_GPG_HOMEDIR}/pubring.gpg}
     else
@@ -437,9 +445,10 @@ if ! /bin/kill -0 "$DBUS_SESSION_BUS_PID"; then
 fi
 
 cleanup () {
-    /bin/kill -9 $DBUS_SESSION_BUS_PID ${FLATPAK_HTTP_PID:-}
+    /bin/kill -9 $DBUS_SESSION_BUS_PID
     gpg-connect-agent --homedir "${FL_GPG_HOMEDIR}" killagent /bye || true
     fusermount -u $XDG_RUNTIME_DIR/doc || :
+    kill $(jobs -p) &> /dev/null || true
     if test -n "${TEST_SKIP_CLEANUP:-}"; then
         echo "Skipping cleanup of ${TEST_DATA_DIR}"
     else

tests/oci-registry-server.py

@@ -7,12 +7,8 @@ import os
 import sys
 import time
 
-if sys.version_info[0] >= 3:
-    from urllib.parse import parse_qs
-    import http.server as http_server
-else:
-    from urllib.parse import parse_qs
-    import http.server as http_server
+from urllib.parse import parse_qs
+import http.server as http_server
 
 repositories = {}
 icons = {}
@@ -246,11 +242,24 @@ class RequestHandler(http_server.BaseHTTPRequestHandler):
             self.end_headers()
             return
 
-def test():
-    if sys.version_info[0] >= 3:
-        http_server.test(RequestHandler, port=0)
-    else:
-        http_server.test(RequestHandler)
+def run(dir):
+    RequestHandler.protocol_version = "HTTP/1.0"
+    httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler)
+    host, port = httpd.socket.getsockname()[:2]
+    with open("httpd-port", 'w') as file:
+        file.write("%d" % port)
+    try:
+        os.write(3, bytes("Started\n", 'utf-8'));
+    except:
+        pass
+    print("Serving HTTP on port %d" % port);
+    if dir:
+        os.chdir(dir)
+    httpd.serve_forever()
 
 if __name__ == '__main__':
-    test()
+    dir = None
+    if len(sys.argv) >= 2 and len(sys.argv[1]) > 0:
+        dir = sys.argv[1]
+
+    run(dir)

tests/test-http-utils.sh

@@ -21,10 +21,8 @@ set -euo pipefail
 
 . $(dirname $0)/libtest.sh
 
-$(dirname $0)/test-webserver.sh "" "python3 $test_srcdir/http-utils-test-server.py 0"
-FLATPAK_HTTP_PID=$(cat httpd-pid)
-mv httpd-port httpd-port-main
-port=$(cat httpd-port-main)
+httpd http-utils-test-server.py .
+port=$(cat httpd-port)
 
 assert_result() {
     test_string=$1

tests/test-oci-registry.sh

@@ -35,10 +35,8 @@ fi
 
 # Start the fake registry server
 
-$(dirname $0)/test-webserver.sh "" "python3 $test_srcdir/oci-registry-server.py 0"
-FLATPAK_HTTP_PID=$(cat httpd-pid)
-mv httpd-port httpd-port-main
-port=$(cat httpd-port-main)
+httpd oci-registry-server.py .
+port=$(cat httpd-port)
 client="python3 $test_srcdir/oci-registry-client.py 127.0.0.1:$port"
 
 setup_repo_no_add oci

tests/test-repo.sh

@@ -50,7 +50,7 @@ if [ x${USE_COLLECTIONS_IN_CLIENT-} == xyes ] ; then
 elif [ x${USE_COLLECTIONS_IN_SERVER-} == xyes ] ; then
     # Set a collection ID and GPG on the server, but not in the client configuration
     setup_repo_no_add test-no-gpg org.test.Collection.NoGpg
-    port=$(cat httpd-port-main)
+    port=$(cat httpd-port)
     flatpak remote-add ${U} --no-gpg-verify test-no-gpg-repo "http://127.0.0.1:${port}/test-no-gpg"
 else
     GPGPUBKEY="" GPGARGS="" setup_repo test-no-gpg
@@ -64,13 +64,13 @@ GPGPUBKEY="${FL_GPG_HOMEDIR2}/pubring.gpg" GPGARGS="${FL_GPGARGS2}" setup_repo t
 #remote with missing GPG key
 # Don’t use --collection-id= here, or the collections code will grab the appropriate
 # GPG key from one of the previously-configured remotes with the same collection ID.
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 if flatpak remote-add ${U} test-missing-gpg-repo "http://127.0.0.1:${port}/test"; then
     assert_not_reached "Should fail metadata-update due to missing gpg key"
 fi
 
 #remote with wrong GPG key
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 if flatpak remote-add ${U} --gpg-import=${FL_GPG_HOMEDIR2}/pubring.gpg test-wrong-gpg-repo "http://127.0.0.1:${port}/test"; then
     assert_not_reached "Should fail metadata-update due to wrong gpg key"
 fi
@@ -170,7 +170,7 @@ fi
 
 echo "ok missing remote name auto-corrects for install"
 
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 if ${FLATPAK} ${U} install -y http://127.0.0.1:${port}/nonexistent.flatpakref 2> install-error-log; then
     assert_not_reached "Should not be able to install a nonexistent flatpakref"
 fi
@@ -184,7 +184,7 @@ setup_repo_no_add flatpakref org.test.Collection.Flatpakref
 cat << EOF > repos/flatpakref/flatpakref-repo.flatpakrepo
 [Flatpak Repo]
 Version=1
-Url=http://127.0.0.1:$(cat httpd-port-main)/flatpakref/
+Url=http://127.0.0.1:$(cat httpd-port)/flatpakref/
 Title=The Title
 GPGKey=${FL_GPG_BASE64}
 EOF
@@ -197,9 +197,9 @@ cat << EOF > org.test.Hello.flatpakref
 [Flatpak Ref]
 Name=org.test.Hello
 Branch=master
-Url=http://127.0.0.1:$(cat httpd-port-main)/flatpakref
+Url=http://127.0.0.1:$(cat httpd-port)/flatpakref
 GPGKey=${FL_GPG_BASE64}
-RuntimeRepo=http://127.0.0.1:$(cat httpd-port-main)/flatpakref/flatpakref-repo.flatpakrepo
+RuntimeRepo=http://127.0.0.1:$(cat httpd-port)/flatpakref/flatpakref-repo.flatpakrepo
 EOF
 
 ${FLATPAK} ${U} uninstall -y org.test.Platform org.test.Hello
@@ -427,7 +427,7 @@ echo "ok eol-rebase"
 
 ${FLATPAK} ${U} install -y test-repo org.test.Platform
 
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 UPDATE_REPO_ARGS="--redirect-url=http://127.0.0.1:${port}/test-gpg3 --gpg-import=${FL_GPG_HOMEDIR2}/pubring.gpg" update_repo
 GPGPUBKEY="${FL_GPG_HOMEDIR2}/pubring.gpg" GPGARGS="${FL_GPGARGS2}" setup_repo_no_add test-gpg3 org.test.Collection.test master
 
@@ -505,7 +505,7 @@ assert_not_file_has_content list-log "org\.test\.Hello"
 assert_not_file_has_content list-log "org\.test\.Platform"
 
 # Disable the remote to make sure we don't do i/o
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 ${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/disable-test" test-repo
 
 ${FLATPAK} ${U} install -y --no-pull test-repo org.test.Hello
@@ -528,7 +528,7 @@ assert_not_file_has_content list-log "org\.test\.Hello"
 assert_not_file_has_content list-log "org\.test\.Platform"
 
 # Disable the remote to make sure we don't do i/o
-port=$(cat httpd-port-main)
+port=$(cat httpd-port)
 ${FLATPAK} ${U} remote-modify --url="http://127.0.0.1:${port}/disable-test" test-repo
 
 # Note: The partial ref is only auto-corrected without user interaction because we're using -y

tests/test-unsigned-summaries.sh

@@ -101,10 +101,10 @@ cat << EOF > org.test.App.flatpakref
 Title=Test App
 Name=org.test.App
 Branch=master
-Url=http://127.0.0.1:$(cat httpd-port-main)/test
+Url=http://127.0.0.1:$(cat httpd-port)/test
 IsRuntime=False
 GPGKey=${FL_GPG_BASE64}
-#RuntimeRepo=http://127.0.0.1:$(cat httpd-port-main)/test
+#RuntimeRepo=http://127.0.0.1:$(cat httpd-port)/test
 DeployCollectionID=org.test.Collection
 EOF
 

tests/test-webserver.sh

@@ -3,33 +3,8 @@
 set -euo pipefail
 
 dir=$1
-cmd=${2:-python3 -m http.server 0}
-test_tmpdir=$(pwd)
 
-[ "$dir" != "" ] && cd ${dir}
-echo "Running web server: PYTHONUNBUFFERED=1 setsid $cmd" >&2
-touch ${test_tmpdir}/httpd-output
-env PYTHONUNBUFFERED=1 setsid $cmd >${test_tmpdir}/httpd-output &
-child_pid=$!
-echo "Web server pid: $child_pid" >&2
-
-for x in $(seq 300); do
-    echo "Waiting for web server ($x/300)..." >&2
-    # Snapshot the output
-    cp ${test_tmpdir}/httpd-output{,.tmp}
-    sed -ne 's/^/# httpd-output.tmp: /' < ${test_tmpdir}/httpd-output.tmp >&2
-    echo >&2
-    # If it's non-empty, see whether it matches our regexp
-    if test -s ${test_tmpdir}/httpd-output.tmp; then
-        sed -e 's,Serving HTTP on 0.0.0.0 port \([0-9]*\) (http://0.0.0.0:[0-9]*/) \.\.\.,\1,' < ${test_tmpdir}/httpd-output.tmp > ${test_tmpdir}/httpd-port
-        if ! cmp ${test_tmpdir}/httpd-output.tmp ${test_tmpdir}/httpd-port 1>/dev/null; then
-            # If so, we've successfully extracted the port
-            break
-        fi
-    fi
-    sleep 0.1
-done
-port=$(cat ${test_tmpdir}/httpd-port)
-echo "http://127.0.0.1:${port}" > ${test_tmpdir}/httpd-address
-echo "$child_pid" > ${test_tmpdir}/httpd-pid
-echo "Started web server '$cmd': process $child_pid on port $port" >&2
+rm -f httpd-pipe
+mkfifo httpd-pipe
+$(dirname $0)/web-server.py "$dir" 3> httpd-pipe &
+read < httpd-pipe

tests/web-server.py

@@ -0,0 +1,64 @@
+#!/usr/bin/python3
+
+from wsgiref.handlers import format_date_time
+from email.utils import parsedate
+from calendar import timegm
+import gzip
+import sys
+import time
+import zlib
+import os
+from http import HTTPStatus
+from urllib.parse import parse_qs
+import http.server as http_server
+from io import BytesIO
+import sys
+
+class RequestHandler(http_server.SimpleHTTPRequestHandler):
+    def handle_tokens(self):
+        need_token_path = self.translate_path(self.path) + ".need_token"
+        if os.path.isfile(need_token_path):
+            with open(need_token_path, 'r') as content_file:
+                token_content = content_file.read()
+            token = None
+            auth = self.headers.get("Authorization")
+            if auth and auth.startswith("Bearer "):
+                token = auth[7:]
+            if token == None:
+                self.send_response(HTTPStatus.UNAUTHORIZED, "No token")
+                self.end_headers()
+                return True
+            if token != token_content:
+                self.send_response(HTTPStatus.UNAUTHORIZED, "Wrong token")
+                self.end_headers()
+                return True
+        return False
+
+    def do_GET(self):
+        if self.handle_tokens():
+            return None
+        return super().do_GET()
+
+def run(dir):
+    RequestHandler.protocol_version = "HTTP/1.0"
+    httpd = http_server.ThreadingHTTPServer( ("127.0.0.1", 0), RequestHandler)
+    host, port = httpd.socket.getsockname()[:2]
+    with open("httpd-port", 'w') as file:
+        file.write("%d" % port)
+    with open("httpd-pid", 'w') as file:
+        file.write("%d" % os.getpid())
+    try:
+        os.write(3, bytes("Started\n", 'utf-8'));
+    except:
+        pass
+    print("Serving HTTP on port %d" % port);
+    if dir:
+        os.chdir(dir)
+    httpd.serve_forever()
+
+if __name__ == '__main__':
+    dir = None
+    if len(sys.argv) >= 2 and len(sys.argv[1]) > 0:
+        dir = sys.argv[1]
+
+    run(dir)