Xen project Mailing List

[qemu-xen master] Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-and-misc-110720-2' into staging

Date: Fri, 30 Oct 2020 21:11:04 +0000

Delivery-date: Fri, 30 Oct 2020 21:11:17 +0000

List-id: "Change log for Mercurial $receive only$" <xen-changelog.lists.xenproject.org>

=== This changeset includes merge from high-traffic branch === Commits on that branch are not reported individually. commit 9f526fce49c6ac48114ed04914b5a76e4db75785 Merge: d34498309cff7560ac90c422c56e3137e6a64b19 4a40f561d5ebb5050a8c6dcbdcee85621056590a Author: Peter Maydell <peter.maydell@xxxxxxxxxx> AuthorDate: Sun Jul 12 15:32:05 2020 +0100 Commit: Peter Maydell <peter.maydell@xxxxxxxxxx> CommitDate: Sun Jul 12 15:32:05 2020 +0100 Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-and-misc-110720-2' into staging Testing and misc build updates: - tests/vm support for aarch64 VMs - tests/tcg better cross-compiler detection - update docker tooling to support registries - update docker support for xtensa - gitlab build docker images and store in registry - gitlab use docker images for builds - a number of skipIf updates to support move - linux-user MAP_FIXED_NOREPLACE fix - qht-bench compiler tweaks - configure fix for secret keyring - tsan fiber annotation clean-up - doc updates for mttcg/icount/gdbstub - fix cirrus to use brew bash for iotests - revert virtio-gpu breakage - fix LC_ALL to avoid sorting changes in iotests # gpg: Signature made Sat 11 Jul 2020 15:56:42 BST # gpg: using RSA key 6685AE99E75167BCAFC8DF35FBD0DB095A9E2A44 # gpg: Good signature from "Alex BennÃ©e (Master Work Key) <alex.bennee@xxxxxxxxxx>" [full] # Primary key fingerprint: 6685 AE99 E751 67BC AFC8 DF35 FBD0 DB09 5A9E 2A44 * remotes/stsquad/tags/pull-testing-and-misc-110720-2: (50 commits) iotests: Set LC_ALL=C for sort Revert "vga: build virtio-gpu as module" tests: fix "make check-qtest" for modular builds .cirrus.yml: add bash to the brew packages tests/docker: update toolchain set in debian-xtensa-cross tests/docker: fall back more gracefully when pull fails docs: Add to gdbstub documentation the PhyMemMode docs/devel: add some notes on tcg-icount for developers docs/devel: convert and update MTTCG design document tests/qht-bench: Adjust threshold computation tests/qht-bench: Adjust testing rate by -1 travis.yml: Test also the other targets on s390x shippable: pull images from registry instead of building testing: add check-build target containers.yml: build with docker.py tooling gitlab: limit re-builds of the containers tests: improve performance of device-introspect-test gitlab: add avocado asset caching gitlab: enable check-tcg for linux-user tests linux-user/elfload: use MAP_FIXED_NOREPLACE in pgb_reserved_va ... Signed-off-by: Peter Maydell <peter.maydell@xxxxxxxxxx> .cirrus.yml | 4 +- .gitignore | 1 + .gitlab-ci.d/containers.yml | 263 +++++++++++++++ .gitlab-ci.d/edk2.yml | 5 +- .gitlab-ci.d/opensbi.yml | 5 +- .gitlab-ci.yml | 265 +++++++++------ .shippable.yml | 8 +- .travis.yml | 62 ++-- configure | 31 +- docs/devel/index.rst | 2 + docs/devel/multi-thread-tcg.rst | 372 +++++++++++++++++++++ docs/devel/multi-thread-tcg.txt | 358 -------------------- docs/devel/tcg-icount.rst | 97 ++++++ docs/system/gdb.rst | 20 ++ hw/display/Makefile.objs | 23 +- linux-user/elfload.c | 10 +- python/qemu/console_socket.py | 110 ++++++ python/qemu/machine.py | 23 +- tests/Makefile.include | 19 +- tests/acceptance/boot_linux.py | 2 + tests/acceptance/linux_initrd.py | 3 + tests/acceptance/machine_mips_malta.py | 3 + tests/acceptance/machine_rx_gdbsim.py | 2 +- tests/acceptance/replay_kernel.py | 2 +- tests/docker/Makefile.include | 17 +- tests/docker/common.rc | 2 +- tests/docker/docker.py | 47 ++- .../dockerfiles/debian-all-test-cross.docker | 53 +++ tests/docker/dockerfiles/debian-alpha-cross.docker | 2 +- tests/docker/dockerfiles/debian-amd64-cross.docker | 2 +- tests/docker/dockerfiles/debian-amd64.docker | 2 +- tests/docker/dockerfiles/debian-arm64-cross.docker | 2 +- .../dockerfiles/debian-arm64-test-cross.docker | 2 +- tests/docker/dockerfiles/debian-armel-cross.docker | 2 +- tests/docker/dockerfiles/debian-armhf-cross.docker | 2 +- tests/docker/dockerfiles/debian-hppa-cross.docker | 2 +- tests/docker/dockerfiles/debian-m68k-cross.docker | 2 +- tests/docker/dockerfiles/debian-mips-cross.docker | 2 +- .../docker/dockerfiles/debian-mips64-cross.docker | 2 +- .../dockerfiles/debian-mips64el-cross.docker | 2 +- .../docker/dockerfiles/debian-mipsel-cross.docker | 2 +- .../docker/dockerfiles/debian-powerpc-cross.docker | 2 +- tests/docker/dockerfiles/debian-ppc64-cross.docker | 2 +- .../docker/dockerfiles/debian-ppc64el-cross.docker | 2 +- .../docker/dockerfiles/debian-riscv64-cross.docker | 2 +- tests/docker/dockerfiles/debian-s390x-cross.docker | 2 +- tests/docker/dockerfiles/debian-sh4-cross.docker | 2 +- .../docker/dockerfiles/debian-sparc64-cross.docker | 2 +- .../docker/dockerfiles/debian-tricore-cross.docker | 2 +- tests/docker/dockerfiles/debian-win32-cross.docker | 2 +- tests/docker/dockerfiles/debian-win64-cross.docker | 2 +- .../docker/dockerfiles/debian-xtensa-cross.docker | 6 +- tests/docker/dockerfiles/debian9-mxe.docker | 2 +- tests/docker/dockerfiles/fedora.docker | 7 + tests/docker/dockerfiles/ubuntu2004.docker | 10 +- tests/qemu-iotests/common.filter | 2 +- tests/qht-bench.c | 40 ++- tests/qtest/Makefile.include | 1 + tests/qtest/device-introspect-test.c | 60 ++-- tests/tcg/Makefile.qemu | 4 +- tests/tcg/configure.sh | 21 +- tests/vm/Makefile.include | 22 ++ tests/vm/aarch64vm.py | 106 ++++++ tests/vm/basevm.py | 344 ++++++++++++++----- tests/vm/centos-8-aarch64.ks | 51 +++ tests/vm/centos.aarch64 | 227 +++++++++++++ tests/vm/conf_example_aarch64.yml | 51 +++ tests/vm/conf_example_x86.yml | 50 +++ tests/vm/fedora | 17 +- tests/vm/freebsd | 16 +- tests/vm/netbsd | 19 +- tests/vm/openbsd | 17 +- tests/vm/ubuntu.aarch64 | 68 ++++ tests/vm/ubuntu.i386 | 46 +-- tests/vm/ubuntuvm.py | 60 ++++ util/coroutine-ucontext.c | 52 ++- util/module.c | 6 - 77 files changed, 2373 insertions(+), 787 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 69342ae031..f287d23c5b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -20,7 +20,7 @@ macos_task: osx_instance: image: mojave-base install_script: - - brew install pkg-config python gnu-sed glib pixman make sdl2 + - brew install pkg-config python gnu-sed glib pixman make sdl2 bash script: - mkdir build - cd build @@ -33,7 +33,7 @@ macos_xcode_task: # this is an alias for the latest Xcode image: mojave-xcode install_script: - - brew install pkg-config gnu-sed glib pixman make sdl2 + - brew install pkg-config gnu-sed glib pixman make sdl2 bash script: - mkdir build - cd build diff --git a/.gitignore b/.gitignore index 90acb4347d..2992d15931 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,7 @@ *.tp *.vr *.d +!/.gitlab-ci.d !/scripts/qemu-guest-agent/fsfreeze-hook.d *.o .sdk diff --git a/.gitlab-ci.d/containers.yml b/.gitlab-ci.d/containers.yml new file mode 100644 index 0000000000..f3c0ca4d61 --- /dev/null +++ b/.gitlab-ci.d/containers.yml @@ -0,0 +1,263 @@ +.container_job_template: &container_job_definition + image: docker:stable + stage: containers + services: + - docker:dind + before_script: + - export TAG="$CI_REGISTRY_IMAGE/qemu/$NAME:latest" + - export COMMON_TAG="$CI_REGISTRY/qemu-project/qemu/$NAME:latest" + - apk add python3 + - docker info + - docker login registry.gitlab.com -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" + script: + - echo "TAG:$TAG" + - echo "COMMON_TAG:$COMMON_TAG" + - docker pull "$TAG" || docker pull "$COMMON_TAG" || true + - ./tests/docker/docker.py --engine docker build + -t "qemu/$NAME" -f "tests/docker/dockerfiles/$NAME.docker" + -r $CI_REGISTRY_IMAGE + - docker tag "qemu/$NAME" "$TAG" + - docker push "$TAG" + after_script: + - docker logout + rules: + - changes: + - .gitlab-ci.d/containers.yml + - tests/docker/* + - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' + - if: '$CI_COMMIT_REF_NAME == "testing/next"' + +amd64-centos7-container: + <<: *container_job_definition + variables: + NAME: centos7 + +amd64-centos8-container: + <<: *container_job_definition + variables: + NAME: centos8 + +amd64-debian10-container: + <<: *container_job_definition + variables: + NAME: debian10 + +amd64-debian11-container: + <<: *container_job_definition + variables: + NAME: debian11 + +amd64-debian9-container: + <<: *container_job_definition + variables: + NAME: debian9 + +amd64-debian9-mxe-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian9-container'] + variables: + NAME: debian9-mxe + +alpha-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-alpha-cross + +amd64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-amd64-cross + +amd64-debian-user-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-all-test-cross + +amd64-debian-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-amd64 + +arm64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-arm64-cross + +arm64-test-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian11-container'] + variables: + NAME: debian-arm64-test-cross + +armel-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-armel-cross + +armhf-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-armhf-cross + +hppa-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-hppa-cross + +m68k-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-m68k-cross + +mips64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips64-cross + +mips64el-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips64el-cross + +mips-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mips-cross + +mipsel-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-mipsel-cross + +powerpc-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-powerpc-cross + +ppc64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-ppc64-cross + +ppc64el-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-ppc64el-cross + +riscv64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-riscv64-cross + +s390x-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-s390x-cross + +sh4-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-sh4-cross + +sparc64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian10-container'] + variables: + NAME: debian-sparc64-cross + +tricore-debian-cross-container: + <<: *container_job_definition + stage: containers-layer2 + needs: ['amd64-debian9-container'] + variables: + NAME: debian-tricore-cross + +win32-debian-cross-container: + <<: *container_job_definition + stage: containers-layer3 + needs: ['amd64-debian9-mxe-container'] + variables: + NAME: debian-win32-cross + +win64-debian-cross-container: + <<: *container_job_definition + stage: containers-layer3 + needs: ['amd64-debian9-mxe-container'] + variables: + NAME: debian-win64-cross + +xtensa-debian-cross-container: + <<: *container_job_definition + variables: + NAME: debian-xtensa-cross + +cris-fedora-cross-container: + <<: *container_job_definition + variables: + NAME: fedora-cris-cross + +amd64-fedora-container: + <<: *container_job_definition + variables: + NAME: fedora + +i386-fedora-cross-container: + <<: *container_job_definition + variables: + NAME: fedora-i386-cross + +amd64-ubuntu1804-container: + <<: *container_job_definition + variables: + NAME: ubuntu1804 + +amd64-ubuntu2004-container: + <<: *container_job_definition + variables: + NAME: ubuntu2004 + +amd64-ubuntu-container: + <<: *container_job_definition + variables: + NAME: ubuntu diff --git a/.gitlab-ci.d/edk2.yml b/.gitlab-ci.d/edk2.yml index 088ba4b43a..e1e0452416 100644 --- a/.gitlab-ci.d/edk2.yml +++ b/.gitlab-ci.d/edk2.yml @@ -1,8 +1,8 @@ docker-edk2: - stage: build + stage: containers rules: # Only run this job when the Dockerfile is modified - changes: - - .gitlab-ci-edk2.yml + - .gitlab-ci.d/edk2.yml - .gitlab-ci.d/edk2/Dockerfile when: always image: docker:19.03.1 @@ -24,6 +24,7 @@ docker-edk2: - docker push $IMAGE_TAG build-edk2: + stage: build rules: # Only run this job when ... - changes: # ... roms/edk2/ is modified (submodule updated) - roms/edk2/* diff --git a/.gitlab-ci.d/opensbi.yml b/.gitlab-ci.d/opensbi.yml index dd051c0124..62088ec5ec 100644 --- a/.gitlab-ci.d/opensbi.yml +++ b/.gitlab-ci.d/opensbi.yml @@ -1,8 +1,8 @@ docker-opensbi: - stage: build + stage: containers rules: # Only run this job when the Dockerfile is modified - changes: - - .gitlab-ci-opensbi.yml + - .gitlab-ci.d/opensbi.yml - .gitlab-ci.d/opensbi/Dockerfile when: always image: docker:19.03.1 @@ -24,6 +24,7 @@ docker-opensbi: - docker push $IMAGE_TAG build-opensbi: + stage: build rules: # Only run this job when ... - changes: # ... roms/opensbi/ is modified (submodule updated) - roms/opensbi/* diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 349c77aa58..5eeba2791b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,127 +1,186 @@ +# Currently we have two build stages after our containers are built: +# - build (for traditional build and test or first stage build) +# - test (for test stages, using build artefacts from a build stage) +stages: + - containers + - containers-layer2 + - containers-layer3 + - build + - test + +# We assume GitLab has it's own caching set up for RPM/APT repositories so we +# just take care of avocado assets here. +cache: + paths: + - $HOME/avocado/data/cache + include: - local: '/.gitlab-ci.d/edk2.yml' - local: '/.gitlab-ci.d/opensbi.yml' + - local: '/.gitlab-ci.d/containers.yml' + +.native_build_job_template: &native_build_job_definition + stage: build + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + before_script: + - JOBS=$(expr $(nproc) + 1) + script: + - mkdir build + - cd build + - if test -n "$TARGETS"; + then + ../configure --enable-werror $CONFIGURE_ARGS --target-list="$TARGETS" ; + else + ../configure --enable-werror $CONFIGURE_ARGS ; + fi + - make -j"$JOBS" + - if test -n "$MAKE_CHECK_ARGS"; + then + make -j"$JOBS" $MAKE_CHECK_ARGS ; + fi + +.native_test_job_template: &native_test_job_definition + stage: test + image: $CI_REGISTRY_IMAGE/qemu/$IMAGE:latest + script: + - cd build + - find . -type f -exec touch {} + + - make $MAKE_CHECK_ARGS + +.post_acceptance_template: &post_acceptance + after_script: + - cd build + - python3 -c 'import json; r = json.load(open("tests/results/latest/results.json")); [print(t["logfile"]) for t in r["tests"] if t["status"] not in ("PASS", "SKIP")]' | xargs cat + - du -chs $HOME/avocado/data/cache + +build-system-ubuntu-main: + <<: *native_build_job_definition + variables: + IMAGE: ubuntu2004 + TARGETS: aarch64-softmmu alpha-softmmu cris-softmmu hppa-softmmu lm32-softmmu + moxie-softmmu microblazeel-softmmu mips64el-softmmu m68k-softmmu ppc-softmmu + riscv64-softmmu sparc-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + paths: + - build + +check-system-ubuntu-main: + <<: *native_test_job_definition + needs: + - job: build-system-ubuntu-main + artifacts: true + variables: + IMAGE: ubuntu2004 + MAKE_CHECK_ARGS: check + +acceptance-system-ubuntu-main: + <<: *native_test_job_definition + needs: + - job: build-system-ubuntu-main + artifacts: true + variables: + IMAGE: ubuntu2004 + MAKE_CHECK_ARGS: check-acceptance + <<: *post_acceptance + +build-system-fedora-alt: + <<: *native_build_job_definition + variables: + IMAGE: fedora + TARGETS: tricore-softmmu unicore32-softmmu microblaze-softmmu mips-softmmu + riscv32-softmmu s390x-softmmu sh4-softmmu sparc64-softmmu x86_64-softmmu + xtensa-softmmu nios2-softmmu or1k-softmmu + MAKE_CHECK_ARGS: check-build + artifacts: + paths: + - build + +check-system-fedora-alt: + <<: *native_test_job_definition + needs: + - job: build-system-fedora-alt + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check -.update_apt_template: &before_script_apt - before_script: - - apt-get update -qq - - apt-get install -y -qq git gcc libglib2.0-dev libpixman-1-dev make - genisoimage - - JOBS=$(expr $(nproc) + 1) - -.update_dnf_template: &before_script_dnf - before_script: - - dnf update -y - - dnf install -y bzip2 diffutils gcc git genisoimage findutils glib2-devel - make python3 perl-podlators perl-Test-Harness pixman-devel zlib-devel - - JOBS=$(expr $(nproc) + 1) - -build-system1: - image: ubuntu:19.10 - <<: *before_script_apt - script: - - apt-get install -y -qq libgtk-3-dev libvte-dev nettle-dev libcacard-dev - libusb-dev libvde-dev libspice-protocol-dev libgl1-mesa-dev libvdeplug-dev - - mkdir build - - cd build - - ../configure --enable-werror --target-list="aarch64-softmmu alpha-softmmu - cris-softmmu hppa-softmmu lm32-softmmu moxie-softmmu microblazeel-softmmu - mips64el-softmmu m68k-softmmu ppc-softmmu riscv64-softmmu sparc-softmmu" - - make -j"$JOBS" - - make -j"$JOBS" check - -build-system2: - image: fedora:latest - <<: *before_script_dnf - script: - - yum install -y SDL2-devel libgcrypt-devel brlapi-devel libaio-devel - libfdt-devel lzo-devel librdmacm-devel libibverbs-devel libibumad-devel - libzstd-devel - - mkdir build - - cd build - - ../configure --enable-werror --target-list="tricore-softmmu unicore32-softmmu - microblaze-softmmu mips-softmmu riscv32-softmmu s390x-softmmu sh4-softmmu - sparc64-softmmu x86_64-softmmu xtensa-softmmu nios2-softmmu or1k-softmmu" - - make -j"$JOBS" - - make -j"$JOBS" check +acceptance-system-fedora-alt: + <<: *native_test_job_definition + needs: + - job: build-system-fedora-alt + artifacts: true + variables: + IMAGE: fedora + MAKE_CHECK_ARGS: check-acceptance + <<: *post_acceptance build-disabled: - image: fedora:latest - <<: *before_script_dnf - script: - - mkdir build - - cd build - - ../configure --enable-werror --disable-rdma --disable-slirp --disable-curl + <<: *native_build_job_definition + variables: + IMAGE: fedora + CONFIGURE_ARGS: --disable-rdma --disable-slirp --disable-curl --disable-capstone --disable-live-block-migration --disable-glusterfs --disable-replication --disable-coroutine-pool --disable-smartcard --disable-guest-agent --disable-curses --disable-libxml2 --disable-tpm --disable-qom-cast-debug --disable-spice --disable-vhost-vsock --disable-vhost-net --disable-vhost-crypto --disable-vhost-user - --target-list="i386-softmmu ppc64-softmmu mips64-softmmu i386-linux-user" - - make -j"$JOBS" - - make -j"$JOBS" check-qtest SPEED=slow + TARGETS: i386-softmmu ppc64-softmmu mips64-softmmu i386-linux-user + MAKE_CHECK_ARGS: check-qtest SPEED=slow build-tcg-disabled: - image: centos:8 - <<: *before_script_dnf - script: - - dnf install -y clang gtk3-devel libusbx-devel libgcrypt-devel - - mkdir build - - cd build - - ../configure --cc=clang --enable-werror --disable-tcg --audio-drv-list="" - - make -j"$JOBS" - - make check-unit - - make check-qapi-schema - - cd tests/qemu-iotests/ - - ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048 + <<: *native_build_job_definition + variables: + IMAGE: centos8 + script: + - mkdir build + - cd build + - ../configure --disable-tcg --audio-drv-list="" + - make -j"$JOBS" + - make check-unit + - make check-qapi-schema + - cd tests/qemu-iotests/ + - ./check -raw 001 002 003 004 005 008 009 010 011 012 021 025 032 033 048 052 063 077 086 101 104 106 113 148 150 151 152 157 159 160 163 170 171 183 184 192 194 197 208 215 221 222 226 227 236 253 277 - - ./check -qcow2 028 051 056 057 058 065 067 068 082 085 091 095 096 102 122 + - ./check -qcow2 028 051 056 057 058 065 067 068 082 085 091 095 096 102 122 124 132 139 142 144 145 151 152 155 157 165 194 196 197 200 202 208 209 215 216 218 222 227 234 246 247 248 250 254 255 257 258 260 261 262 263 264 270 272 273 277 279 build-user: - <<: *before_script_apt - script: - - mkdir build - - cd build - - ../configure --enable-werror --disable-system --disable-guest-agent - --disable-capstone --disable-slirp --disable-fdt - - make -j"$JOBS" - - make run-tcg-tests-i386-linux-user run-tcg-tests-x86_64-linux-user + <<: *native_build_job_definition + variables: + IMAGE: debian-all-test-cross + CONFIGURE_ARGS: --disable-tools --disable-system + MAKE_CHECK_ARGS: check-tcg build-clang: - image: fedora:latest - <<: *before_script_dnf - script: - - yum install -y clang SDL2-devel libattr-devel libcap-ng-devel xfsprogs-devel - libiscsi-devel libnfs-devel libseccomp-devel gnutls-devel librbd-devel - - mkdir build - - cd build - - ../configure --cc=clang --cxx=clang++ --enable-werror - --target-list="alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu - ppc-softmmu s390x-softmmu x86_64-softmmu arm-linux-user" - - make -j"$JOBS" - - make -j"$JOBS" check + <<: *native_build_job_definition + variables: + IMAGE: fedora + CONFIGURE_ARGS: --cc=clang --cxx=clang++ + TARGETS: alpha-softmmu arm-softmmu m68k-softmmu mips64-softmmu + ppc-softmmu s390x-softmmu x86_64-softmmu arm-linux-user + MAKE_CHECK_ARGS: check build-tci: - image: centos:8 - <<: *before_script_dnf - script: - - TARGETS="aarch64 alpha arm hppa m68k microblaze moxie ppc64 s390x x86_64" - - mkdir build - - cd build - - ../configure --enable-tcg-interpreter - --target-list="$(for tg in $TARGETS; do echo -n ${tg}'-softmmu '; done)" - - make -j"$JOBS" - - make run-tcg-tests-x86_64-softmmu - - make tests/qtest/boot-serial-test tests/qtest/cdrom-test tests/qtest/pxe-test - - for tg in $TARGETS ; do - export QTEST_QEMU_BINARY="${tg}-softmmu/qemu-system-${tg}" ; - ./tests/qtest/boot-serial-test || exit 1 ; - ./tests/qtest/cdrom-test || exit 1 ; - done - - QTEST_QEMU_BINARY="x86_64-softmmu/qemu-system-x86_64" ./tests/qtest/pxe-test - - QTEST_QEMU_BINARY="s390x-softmmu/qemu-system-s390x" - ./tests/qtest/pxe-test -m slow + <<: *native_build_job_definition + variables: + IMAGE: fedora + script: + - TARGETS="aarch64 alpha arm hppa m68k microblaze moxie ppc64 s390x x86_64" + - mkdir build + - cd build + - ../configure --enable-tcg-interpreter + --target-list="$(for tg in $TARGETS; do echo -n ${tg}'-softmmu '; done)" + - make -j"$JOBS" + - make run-tcg-tests-x86_64-softmmu + - make tests/qtest/boot-serial-test tests/qtest/cdrom-test tests/qtest/pxe-test + - for tg in $TARGETS ; do + export QTEST_QEMU_BINARY="${tg}-softmmu/qemu-system-${tg}" ; + ./tests/qtest/boot-serial-test || exit 1 ; + ./tests/qtest/cdrom-test || exit 1 ; + done + - QTEST_QEMU_BINARY="x86_64-softmmu/qemu-system-x86_64" ./tests/qtest/pxe-test + - QTEST_QEMU_BINARY="s390x-softmmu/qemu-system-s390x" ./tests/qtest/pxe-test -m slow diff --git a/.shippable.yml b/.shippable.yml index 2cce7b5689..f6b742432e 100644 --- a/.shippable.yml +++ b/.shippable.yml @@ -26,12 +26,10 @@ env: - IMAGE=debian-ppc64el-cross TARGET_LIST=ppc64-softmmu,ppc64-linux-user,ppc64abi32-linux-user build: - pre_ci: - - make docker-image-${IMAGE} V=1 pre_ci_boot: - image_name: qemu - image_tag: ${IMAGE} - pull: false + image_name: registry.gitlab.com/qemu-project/qemu/${IMAGE} + image_tag: latest + pull: true options: "-e HOME=/root" ci: - unset CC diff --git a/.travis.yml b/.travis.yml index 74158f741b..ab429500fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -289,29 +289,6 @@ jobs: python: 3.6 - # Acceptance (Functional) tests - - name: "GCC check-acceptance" - dist: bionic - env: - - CONFIG="--enable-tools --target-list=aarch64-softmmu,alpha-softmmu,arm-softmmu,m68k-softmmu,microblaze-softmmu,mips-softmmu,mips64el-softmmu,nios2-softmmu,or1k-softmmu,ppc-softmmu,ppc64-softmmu,s390x-softmmu,sh4-softmmu,sparc-softmmu,x86_64-softmmu,xtensa-softmmu" - - TEST_CMD="make check-acceptance" - - CACHE_NAME="${TRAVIS_BRANCH}-linux-gcc-acceptance" - after_script: - - python3 -c 'import json; r = json.load(open("tests/results/latest/results.json")); [print(t["logfile"]) for t in r["tests"] if t["status"] not in ("PASS", "SKIP")]' | xargs cat - - du -chs $HOME/avocado/data/cache - addons: - apt: - packages: - - python3-pil - - python3-pip - - python3-numpy - - python3-opencv - - python3-venv - - rpm2cpio - - tesseract-ocr - - tesseract-ocr-eng - - # Using newer GCC with sanitizers - name: "GCC9 with sanitizers (softmmu)" addons: @@ -505,6 +482,45 @@ jobs: $(exit $BUILD_RC); fi + - name: "[s390x] GCC (other-softmmu)" + arch: s390x + dist: bionic + addons: + apt_packages: + - libaio-dev + - libattr1-dev + - libcap-ng-dev + - libgnutls28-dev + - libiscsi-dev + - liblttng-ust-dev + - liblzo2-dev + - libncurses-dev + - libnfs-dev + - libnss3-dev + - libpixman-1-dev + - libsdl2-dev + - libsdl2-image-dev + - libseccomp-dev + - libsnappy-dev + - libzstd-dev + - nettle-dev + - xfslibs-dev + # Tests dependencies + - genisoimage + env: + - CONFIG="--disable-containers --audio-drv-list=sdl --disable-user + --target-list-exclude=${MAIN_SOFTMMU_TARGETS}" + + - name: "[s390x] GCC (user)" + arch: s390x + dist: bionic + addons: + apt_packages: + - libgcrypt20-dev + - libgnutls28-dev + env: + - CONFIG="--disable-containers --disable-system" + - name: "[s390x] Clang (disable-tcg)" arch: s390x dist: bionic diff --git a/configure b/configure index 31e2ddbf28..814ed81279 100755 --- a/configure +++ b/configure @@ -418,6 +418,7 @@ prefix="/usr/local" mandir="\${prefix}/share/man" datadir="\${prefix}/share" firmwarepath="\${prefix}/share/qemu-firmware" +efi_aarch64="" qemu_docdir="\${prefix}/share/doc/qemu" bindir="\${prefix}/bin" libdir="\${prefix}/lib" @@ -960,6 +961,13 @@ do fi done +# Check for existence of python3 yaml, needed to +# import yaml config files into vm-build. +python_yaml="no" +if $(python3 -c "import yaml" 2> /dev/null); then + python_yaml="yes" +fi + : ${smbd=${SMBD-/usr/sbin/smbd}} # Default objcc to clang if available, otherwise use CC @@ -1102,6 +1110,8 @@ for opt do ;; --firmwarepath=*) firmwarepath="$optarg" ;; + --efi-aarch64=*) efi_aarch64="$optarg" + ;; --host=*|--build=*|\ --disable-dependency-tracking|\ --sbindir=*|--sharedstatedir=*|\ @@ -1784,6 +1794,7 @@ Advanced options (experts only): --sysconfdir=PATH install config in PATH$confsuffix --localstatedir=PATH install local state in PATH (set at runtime on win32) --firmwarepath=PATH search PATH for firmware files + --efi-aarch64=PATH PATH of efi file to use for aarch64 VMs. --with-confsuffix=SUFFIX suffix for QEMU data inside datadir/libdir/sysconfdir [$confsuffix] --with-pkgversion=VERS use specified string as sub-version of the package --enable-debug enable common debug build options @@ -3620,6 +3631,20 @@ EOF fi fi +############################################ +# efi-aarch64 probe +# Check for efi files needed by aarch64 VMs. +# By default we will use the efi included with QEMU. +# Allow user to override the path for efi also. +if ! test -f "$efi_aarch64"; then + if test -f $source_path/pc-bios/edk2-aarch64-code.fd.bz2; then + # valid after build + efi_aarch64=$PWD/pc-bios/edk2-aarch64-code.fd + else + efi_aarch64="" + fi +fi + ########################################## # libcap-ng library probe if test "$cap_ng" != "no" ; then @@ -6486,7 +6511,7 @@ EOF fi if test "$secret_keyring" != "no" then - if test "$have_keyring" == "yes" + if test "$have_keyring" = "yes" then secret_keyring=yes else @@ -6868,6 +6893,8 @@ if test "$docs" != "no"; then echo "sphinx-build $sphinx_build" fi echo "genisoimage $genisoimage" +echo "efi_aarch64 $efi_aarch64" +echo "python_yaml $python_yaml" echo "slirp support $slirp $(echo_version $slirp $slirp_version)" if test "$slirp" != "no" ; then echo "smbd $smbd" @@ -7966,6 +7993,8 @@ echo "PYTHON=$python" >> $config_host_mak echo "SPHINX_BUILD=$sphinx_build" >> $config_host_mak echo "SPHINX_WERROR=$sphinx_werror" >> $config_host_mak echo "GENISOIMAGE=$genisoimage" >> $config_host_mak +echo "EFI_AARCH64=$efi_aarch64" >> $config_host_mak +echo "PYTHON_YAML=$python_yaml" >> $config_host_mak echo "CC=$cc" >> $config_host_mak if $iasl -h > /dev/null 2>&1; then echo "IASL=$iasl" >> $config_host_mak diff --git a/docs/devel/index.rst b/docs/devel/index.rst index bb8238c5d6..ae6eac7c9c 100644 --- a/docs/devel/index.rst +++ b/docs/devel/index.rst @@ -23,6 +23,8 @@ Contents: decodetree secure-coding-practices tcg + tcg-icount + multi-thread-tcg tcg-plugins bitops reset diff --git a/docs/devel/multi-thread-tcg.rst b/docs/devel/multi-thread-tcg.rst new file mode 100644 index 0000000000..42158b77c7 --- /dev/null +++ b/docs/devel/multi-thread-tcg.rst @@ -0,0 +1,372 @@ +.. + Copyright (c) 2015-2020 Linaro Ltd. + + This work is licensed under the terms of the GNU GPL, version 2 or + later. See the COPYING file in the top-level directory. + +Introduction +============ + +This document outlines the design for multi-threaded TCG (a.k.a MTTCG) +system-mode emulation. user-mode emulation has always mirrored the +thread structure of the translated executable although some of the +changes done for MTTCG system emulation have improved the stability of +linux-user emulation. + +The original system-mode TCG implementation was single threaded and +dealt with multiple CPUs with simple round-robin scheduling. This +simplified a lot of things but became increasingly limited as systems +being emulated gained additional cores and per-core performance gains +for host systems started to level off. + +vCPU Scheduling +=============== + +We introduce a new running mode where each vCPU will run on its own +user-space thread. This is enabled by default for all FE/BE +combinations where the host memory model is able to accommodate the +guest (TCG_GUEST_DEFAULT_MO & ~TCG_TARGET_DEFAULT_MO is zero) and the +guest has had the required work done to support this safely +(TARGET_SUPPORTS_MTTCG). + +System emulation will fall back to the original round robin approach +if: + +* forced by --accel tcg,thread=single +* enabling --icount mode +* 64 bit guests on 32 bit hosts (TCG_OVERSIZED_GUEST) + +In the general case of running translated code there should be no +inter-vCPU dependencies and all vCPUs should be able to run at full +speed. Synchronisation will only be required while accessing internal +shared data structures or when the emulated architecture requires a +coherent representation of the emulated machine state. + +Shared Data Structures +====================== + +Main Run Loop +------------- + +Even when there is no code being generated there are a number of +structures associated with the hot-path through the main run-loop. +These are associated with looking up the next translation block to +execute. These include: + + tb_jmp_cache (per-vCPU, cache of recent jumps) + tb_ctx.htable (global hash table, phys address->tb lookup) + +As TB linking only occurs when blocks are in the same page this code +is critical to performance as looking up the next TB to execute is the +most common reason to exit the generated code. + +DESIGN REQUIREMENT: Make access to lookup structures safe with +multiple reader/writer threads. Minimise any lock contention to do it. + +The hot-path avoids using locks where possible. The tb_jmp_cache is +updated with atomic accesses to ensure consistent results. The fall +back QHT based hash table is also designed for lockless lookups. Locks +are only taken when code generation is required or TranslationBlocks +have their block-to-block jumps patched. + +Global TCG State +---------------- + +User-mode emulation +~~~~~~~~~~~~~~~~~~~ + +We need to protect the entire code generation cycle including any post +generation patching of the translated code. This also implies a shared +translation buffer which contains code running on all cores. Any +execution path that comes to the main run loop will need to hold a +mutex for code generation. This also includes times when we need flush +code or entries from any shared lookups/caches. Structures held on a +per-vCPU basis won't need locking unless other vCPUs will need to +modify them. + +DESIGN REQUIREMENT: Add locking around all code generation and TB +patching. + +(Current solution) + +Code generation is serialised with mmap_lock(). + +!User-mode emulation +~~~~~~~~~~~~~~~~~~~~ + +Each vCPU has its own TCG context and associated TCG region, thereby +requiring no locking during translation. + +Translation Blocks +------------------ + +Currently the whole system shares a single code generation buffer +which when full will force a flush of all translations and start from +scratch again. Some operations also force a full flush of translations +including: + + - debugging operations (breakpoint insertion/removal) + - some CPU helper functions + - linux-user spawning it's first thread + +This is done with the async_safe_run_on_cpu() mechanism to ensure all +vCPUs are quiescent when changes are being made to shared global +structures. + +More granular translation invalidation events are typically due +to a change of the state of a physical page: + + - code modification (self modify code, patching code) + - page changes (new page mapping in linux-user mode) + +While setting the invalid flag in a TranslationBlock will stop it +being used when looked up in the hot-path there are a number of other +book-keeping structures that need to be safely cleared. + +Any TranslationBlocks which have been patched to jump directly to the +now invalid blocks need the jump patches reversing so they will return +to the C code. + +There are a number of look-up caches that need to be properly updated +including the: + + - jump lookup cache + - the physical-to-tb lookup hash table + - the global page table + +The global page table (l1_map) which provides a multi-level look-up +for PageDesc structures which contain pointers to the start of a +linked list of all Translation Blocks in that page (see page_next). + +Both the jump patching and the page cache involve linked lists that +the invalidated TranslationBlock needs to be removed from. + +DESIGN REQUIREMENT: Safely handle invalidation of TBs + - safely patch/revert direct jumps + - remove central PageDesc lookup entries + - ensure lookup caches/hashes are safely updated + +(Current solution) + +The direct jump themselves are updated atomically by the TCG +tb_set_jmp_target() code. Modification to the linked lists that allow +searching for linked pages are done under the protection of tb->jmp_lock, +where tb is the destination block of a jump. Each origin block keeps a +pointer to its destinations so that the appropriate lock can be acquired before +iterating over a jump list. + +The global page table is a lockless radix tree; cmpxchg is used +to atomically insert new elements. + +The lookup caches are updated atomically and the lookup hash uses QHT +which is designed for concurrent safe lookup. + +Parallel code generation is supported. QHT is used at insertion time +as the synchronization point across threads, thereby ensuring that we only +keep track of a single TranslationBlock for each guest code block. + +Memory maps and TLBs +-------------------- + +The memory handling code is fairly critical to the speed of memory +access in the emulated system. The SoftMMU code is designed so the +hot-path can be handled entirely within translated code. This is +handled with a per-vCPU TLB structure which once populated will allow +a series of accesses to the page to occur without exiting the +translated code. It is possible to set flags in the TLB address which +will ensure the slow-path is taken for each access. This can be done +to support: + + - Memory regions (dividing up access to PIO, MMIO and RAM) + - Dirty page tracking (for code gen, SMC detection, migration and display) + - Virtual TLB (for translating guest address->real address) + +When the TLB tables are updated by a vCPU thread other than their own +we need to ensure it is done in a safe way so no inconsistent state is +seen by the vCPU thread. + +Some operations require updating a number of vCPUs TLBs at the same +time in a synchronised manner. + +DESIGN REQUIREMENTS: + + - TLB Flush All/Page + - can be across-vCPUs + - cross vCPU TLB flush may need other vCPU brought to halt + - change may need to be visible to the calling vCPU immediately + - TLB Flag Update + - usually cross-vCPU + - want change to be visible as soon as possible + - TLB Update (update a CPUTLBEntry, via tlb_set_page_with_attrs) + - This is a per-vCPU table - by definition can't race + - updated by its own thread when the slow-path is forced + +(Current solution) + +We have updated cputlb.c to defer operations when a cross-vCPU +operation with async_run_on_cpu() which ensures each vCPU sees a +coherent state when it next runs its work (in a few instructions +time). + +A new set up operations (tlb_flush_*_all_cpus) take an additional flag +which when set will force synchronisation by setting the source vCPUs +work as "safe work" and exiting the cpu run loop. This ensure by the +time execution restarts all flush operations have completed. + +TLB flag updates are all done atomically and are also protected by the +corresponding page lock. + +(Known limitation) + +Not really a limitation but the wait mechanism is overly strict for +some architectures which only need flushes completed by a barrier +instruction. This could be a future optimisation. + +Emulated hardware state +----------------------- + +Currently thanks to KVM work any access to IO memory is automatically +protected by the global iothread mutex, also known as the BQL (Big +Qemu Lock). Any IO region that doesn't use global mutex is expected to +do its own locking. + +However IO memory isn't the only way emulated hardware state can be +modified. Some architectures have model specific registers that +trigger hardware emulation features. Generally any translation helper +that needs to update more than a single vCPUs of state should take the +BQL. + +As the BQL, or global iothread mutex is shared across the system we +push the use of the lock as far down into the TCG code as possible to +minimise contention. + +(Current solution) + +MMIO access automatically serialises hardware emulation by way of the +BQL. Currently Arm targets serialise all ARM_CP_IO register accesses +and also defer the reset/startup of vCPUs to the vCPU context by way +of async_run_on_cpu(). + +Updates to interrupt state are also protected by the BQL as they can +often be cross vCPU. + +Memory Consistency +================== + +Between emulated guests and host systems there are a range of memory +consistency models. Even emulating weakly ordered systems on strongly +ordered hosts needs to ensure things like store-after-load re-ordering +can be prevented when the guest wants to. + +Memory Barriers +--------------- + +Barriers (sometimes known as fences) provide a mechanism for software +to enforce a particular ordering of memory operations from the point +of view of external observers (e.g. another processor core). They can +apply to any memory operations as well as just loads or stores. + +The Linux kernel has an excellent `write-up +<https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/Documentation/memory-barriers.txt>` +on the various forms of memory barrier and the guarantees they can +provide. + +Barriers are often wrapped around synchronisation primitives to +provide explicit memory ordering semantics. However they can be used +by themselves to provide safe lockless access by ensuring for example +a change to a signal flag will only be visible once the changes to +payload are. + +DESIGN REQUIREMENT: Add a new tcg_memory_barrier op + +This would enforce a strong load/store ordering so all loads/stores +complete at the memory barrier. On single-core non-SMP strongly +ordered backends this could become a NOP. + +Aside from explicit standalone memory barrier instructions there are +also implicit memory ordering semantics which comes with each guest +memory access instruction. For example all x86 load/stores come with +fairly strong guarantees of sequential consistency whereas Arm has +special variants of load/store instructions that imply acquire/release +semantics. + +In the case of a strongly ordered guest architecture being emulated on +a weakly ordered host the scope for a heavy performance impact is +quite high. + +DESIGN REQUIREMENTS: Be efficient with use of memory barriers + - host systems with stronger implied guarantees can skip some barriers + - merge consecutive barriers to the strongest one + +(Current solution) + +The system currently has a tcg_gen_mb() which will add memory barrier +operations if code generation is being done in a parallel context. The +tcg_optimize() function attempts to merge barriers up to their +strongest form before any load/store operations. The solution was +originally developed and tested for linux-user based systems. All +backends have been converted to emit fences when required. So far the +following front-ends have been updated to emit fences when required: + + - target-i386 + - target-arm + - target-aarch64 + - target-alpha + - target-mips + +Memory Control and Maintenance +------------------------------ + +This includes a class of instructions for controlling system cache +behaviour. While QEMU doesn't model cache behaviour these instructions +are often seen when code modification has taken place to ensure the +changes take effect. + +Synchronisation Primitives +-------------------------- + +There are two broad types of synchronisation primitives found in +modern ISAs: atomic instructions and exclusive regions. + +The first type offer a simple atomic instruction which will guarantee +some sort of test and conditional store will be truly atomic w.r.t. +other cores sharing access to the memory. The classic example is the +x86 cmpxchg instruction. + +The second type offer a pair of load/store instructions which offer a +guarantee that a region of memory has not been touched between the +load and store instructions. An example of this is Arm's ldrex/strex +pair where the strex instruction will return a flag indicating a +successful store only if no other CPU has accessed the memory region +since the ldrex. + +Traditionally TCG has generated a series of operations that work +because they are within the context of a single translation block so +will have completed before another CPU is scheduled. However with +the ability to have multiple threads running to emulate multiple CPUs +we will need to explicitly expose these semantics. + +DESIGN REQUIREMENTS: + - Support classic atomic instructions + - Support load/store exclusive (or load link/store conditional) pairs + - Generic enough infrastructure to support all guest architectures +CURRENT OPEN QUESTIONS: + - How problematic is the ABA problem in general? + +(Current solution) + +The TCG provides a number of atomic helpers (tcg_gen_atomic_*) which +can be used directly or combined to emulate other instructions like +Arm's ldrex/strex instructions. While they are susceptible to the ABA +problem so far common guests have not implemented patterns where +this may be a problem - typically presenting a locking ABI which +assumes cmpxchg like semantics. + +The code also includes a fall-back for cases where multi-threaded TCG +ops can't work (e.g. guest atomic width > host atomic width). In this +case an EXCP_ATOMIC exit occurs and the instruction is emulated with +an exclusive lock which ensures all emulation is serialised. + +While the atomic helpers look good enough for now there may be a need +to look at solutions that can more closely model the guest +architectures semantics. diff --git a/docs/devel/multi-thread-tcg.txt b/docs/devel/multi-thread-tcg.txt deleted file mode 100644 index 3c85ac0eab..0000000000 --- a/docs/devel/multi-thread-tcg.txt +++ /dev/null @@ -1,358 +0,0 @@ -Copyright (c) 2015-2016 Linaro Ltd. - -This work is licensed under the terms of the GNU GPL, version 2 or -later. See the COPYING file in the top-level directory. - -Introduction -============ - -This document outlines the design for multi-threaded TCG system-mode -emulation. The current user-mode emulation mirrors the thread -structure of the translated executable. Some of the work will be -applicable to both system and linux-user emulation. - -The original system-mode TCG implementation was single threaded and -dealt with multiple CPUs with simple round-robin scheduling. This -simplified a lot of things but became increasingly limited as systems -being emulated gained additional cores and per-core performance gains -for host systems started to level off. - -vCPU Scheduling -=============== - -We introduce a new running mode where each vCPU will run on its own -user-space thread. This will be enabled by default for all FE/BE -combinations that have had the required work done to support this -safely. - -In the general case of running translated code there should be no -inter-vCPU dependencies and all vCPUs should be able to run at full -speed. Synchronisation will only be required while accessing internal -shared data structures or when the emulated architecture requires a -coherent representation of the emulated machine state. - -Shared Data Structures -====================== - -Main Run Loop -------------- - -Even when there is no code being generated there are a number of -structures associated with the hot-path through the main run-loop. -These are associated with looking up the next translation block to -execute. These include: - - tb_jmp_cache (per-vCPU, cache of recent jumps) - tb_ctx.htable (global hash table, phys address->tb lookup) - -As TB linking only occurs when blocks are in the same page this code -is critical to performance as looking up the next TB to execute is the -most common reason to exit the generated code. - -DESIGN REQUIREMENT: Make access to lookup structures safe with -multiple reader/writer threads. Minimise any lock contention to do it. - -The hot-path avoids using locks where possible. The tb_jmp_cache is -updated with atomic accesses to ensure consistent results. The fall -back QHT based hash table is also designed for lockless lookups. Locks -are only taken when code generation is required or TranslationBlocks -have their block-to-block jumps patched. - -Global TCG State ----------------- - -### User-mode emulation -We need to protect the entire code generation cycle including any post -generation patching of the translated code. This also implies a shared -translation buffer which contains code running on all cores. Any -execution path that comes to the main run loop will need to hold a -mutex for code generation. This also includes times when we need flush -code or entries from any shared lookups/caches. Structures held on a -per-vCPU basis won't need locking unless other vCPUs will need to -modify them. - -DESIGN REQUIREMENT: Add locking around all code generation and TB -patching. - -(Current solution) - -Code generation is serialised with mmap_lock(). - -### !User-mode emulation -Each vCPU has its own TCG context and associated TCG region, thereby -requiring no locking. - -Translation Blocks ------------------- - -Currently the whole system shares a single code generation buffer -which when full will force a flush of all translations and start from -scratch again. Some operations also force a full flush of translations -including: - - - debugging operations (breakpoint insertion/removal) - - some CPU helper functions - -This is done with the async_safe_run_on_cpu() mechanism to ensure all -vCPUs are quiescent when changes are being made to shared global -structures. - -More granular translation invalidation events are typically due -to a change of the state of a physical page: - - - code modification (self modify code, patching code) - - page changes (new page mapping in linux-user mode) - -While setting the invalid flag in a TranslationBlock will stop it -being used when looked up in the hot-path there are a number of other -book-keeping structures that need to be safely cleared. - -Any TranslationBlocks which have been patched to jump directly to the -now invalid blocks need the jump patches reversing so they will return -to the C code. - -There are a number of look-up caches that need to be properly updated -including the: - - - jump lookup cache - - the physical-to-tb lookup hash table - - the global page table - -The global page table (l1_map) which provides a multi-level look-up -for PageDesc structures which contain pointers to the start of a -linked list of all Translation Blocks in that page (see page_next). - -Both the jump patching and the page cache involve linked lists that -the invalidated TranslationBlock needs to be removed from. - -DESIGN REQUIREMENT: Safely handle invalidation of TBs - - safely patch/revert direct jumps - - remove central PageDesc lookup entries - - ensure lookup caches/hashes are safely updated - -(Current solution) - -The direct jump themselves are updated atomically by the TCG -tb_set_jmp_target() code. Modification to the linked lists that allow -searching for linked pages are done under the protection of tb->jmp_lock, -where tb is the destination block of a jump. Each origin block keeps a -pointer to its destinations so that the appropriate lock can be acquired before -iterating over a jump list. - -The global page table is a lockless radix tree; cmpxchg is used -to atomically insert new elements. - -The lookup caches are updated atomically and the lookup hash uses QHT -which is designed for concurrent safe lookup. - -Parallel code generation is supported. QHT is used at insertion time -as the synchronization point across threads, thereby ensuring that we only -keep track of a single TranslationBlock for each guest code block. - -Memory maps and TLBs --------------------- - -The memory handling code is fairly critical to the speed of memory -access in the emulated system. The SoftMMU code is designed so the -hot-path can be handled entirely within translated code. This is -handled with a per-vCPU TLB structure which once populated will allow -a series of accesses to the page to occur without exiting the -translated code. It is possible to set flags in the TLB address which -will ensure the slow-path is taken for each access. This can be done -to support: - - - Memory regions (dividing up access to PIO, MMIO and RAM) - - Dirty page tracking (for code gen, SMC detection, migration and display) - - Virtual TLB (for translating guest address->real address) - -When the TLB tables are updated by a vCPU thread other than their own -we need to ensure it is done in a safe way so no inconsistent state is -seen by the vCPU thread. - -Some operations require updating a number of vCPUs TLBs at the same -time in a synchronised manner. - -DESIGN REQUIREMENTS: - - - TLB Flush All/Page - - can be across-vCPUs - - cross vCPU TLB flush may need other vCPU brought to halt - - change may need to be visible to the calling vCPU immediately - - TLB Flag Update - - usually cross-vCPU - - want change to be visible as soon as possible - - TLB Update (update a CPUTLBEntry, via tlb_set_page_with_attrs) - - This is a per-vCPU table - by definition can't race - - updated by its own thread when the slow-path is forced - -(Current solution) - -We have updated cputlb.c to defer operations when a cross-vCPU -operation with async_run_on_cpu() which ensures each vCPU sees a -coherent state when it next runs its work (in a few instructions -time). - -A new set up operations (tlb_flush_*_all_cpus) take an additional flag -which when set will force synchronisation by setting the source vCPUs -work as "safe work" and exiting the cpu run loop. This ensure by the -time execution restarts all flush operations have completed. - -TLB flag updates are all done atomically and are also protected by the -corresponding page lock. - -(Known limitation) - -Not really a limitation but the wait mechanism is overly strict for -some architectures which only need flushes completed by a barrier -instruction. This could be a future optimisation. - -Emulated hardware state ------------------------ - -Currently thanks to KVM work any access to IO memory is automatically -protected by the global iothread mutex, also known as the BQL (Big -Qemu Lock). Any IO region that doesn't use global mutex is expected to -do its own locking. - -However IO memory isn't the only way emulated hardware state can be -modified. Some architectures have model specific registers that -trigger hardware emulation features. Generally any translation helper -that needs to update more than a single vCPUs of state should take the -BQL. - -As the BQL, or global iothread mutex is shared across the system we -push the use of the lock as far down into the TCG code as possible to -minimise contention. - -(Current solution) - -MMIO access automatically serialises hardware emulation by way of the -BQL. Currently Arm targets serialise all ARM_CP_IO register accesses -and also defer the reset/startup of vCPUs to the vCPU context by way -of async_run_on_cpu(). - -Updates to interrupt state are also protected by the BQL as they can -often be cross vCPU. - -Memory Consistency -================== - -Between emulated guests and host systems there are a range of memory -consistency models. Even emulating weakly ordered systems on strongly -ordered hosts needs to ensure things like store-after-load re-ordering -can be prevented when the guest wants to. - -Memory Barriers ---------------- - -Barriers (sometimes known as fences) provide a mechanism for software -to enforce a particular ordering of memory operations from the point -of view of external observers (e.g. another processor core). They can -apply to any memory operations as well as just loads or stores. - -The Linux kernel has an excellent write-up on the various forms of -memory barrier and the guarantees they can provide [1]. - -Barriers are often wrapped around synchronisation primitives to -provide explicit memory ordering semantics. However they can be used -by themselves to provide safe lockless access by ensuring for example -a change to a signal flag will only be visible once the changes to -payload are. - -DESIGN REQUIREMENT: Add a new tcg_memory_barrier op - -This would enforce a strong load/store ordering so all loads/stores -complete at the memory barrier. On single-core non-SMP strongly -ordered backends this could become a NOP. - -Aside from explicit standalone memory barrier instructions there are -also implicit memory ordering semantics which comes with each guest -memory access instruction. For example all x86 load/stores come with -fairly strong guarantees of sequential consistency whereas Arm has -special variants of load/store instructions that imply acquire/release -semantics. - -In the case of a strongly ordered guest architecture being emulated on -a weakly ordered host the scope for a heavy performance impact is -quite high. - -DESIGN REQUIREMENTS: Be efficient with use of memory barriers - - host systems with stronger implied guarantees can skip some barriers - - merge consecutive barriers to the strongest one - -(Current solution) - -The system currently has a tcg_gen_mb() which will add memory barrier -operations if code generation is being done in a parallel context. The -tcg_optimize() function attempts to merge barriers up to their -strongest form before any load/store operations. The solution was -originally developed and tested for linux-user based systems. All -backends have been converted to emit fences when required. So far the -following front-ends have been updated to emit fences when required: - - - target-i386 - - target-arm - - target-aarch64 - - target-alpha - - target-mips - -Memory Control and Maintenance ------------------------------- - -This includes a class of instructions for controlling system cache -behaviour. While QEMU doesn't model cache behaviour these instructions -are often seen when code modification has taken place to ensure the -changes take effect. - -Synchronisation Primitives --------------------------- - -There are two broad types of synchronisation primitives found in -modern ISAs: atomic instructions and exclusive regions. - -The first type offer a simple atomic instruction which will guarantee -some sort of test and conditional store will be truly atomic w.r.t. -other cores sharing access to the memory. The classic example is the -x86 cmpxchg instruction. - -The second type offer a pair of load/store instructions which offer a -guarantee that a region of memory has not been touched between the -load and store instructions. An example of this is Arm's ldrex/strex -pair where the strex instruction will return a flag indicating a -successful store only if no other CPU has accessed the memory region -since the ldrex. - -Traditionally TCG has generated a series of operations that work -because they are within the context of a single translation block so -will have completed before another CPU is scheduled. However with -the ability to have multiple threads running to emulate multiple CPUs -we will need to explicitly expose these semantics. - -DESIGN REQUIREMENTS: - - Support classic atomic instructions - - Support load/store exclusive (or load link/store conditional) pairs - - Generic enough infrastructure to support all guest architectures -CURRENT OPEN QUESTIONS: - - How problematic is the ABA problem in general? - -(Current solution) - -The TCG provides a number of atomic helpers (tcg_gen_atomic_*) which -can be used directly or combined to emulate other instructions like -Arm's ldrex/strex instructions. While they are susceptible to the ABA -problem so far common guests have not implemented patterns where -this may be a problem - typically presenting a locking ABI which -assumes cmpxchg like semantics. - -The code also includes a fall-back for cases where multi-threaded TCG -ops can't work (e.g. guest atomic width > host atomic width). In this -case an EXCP_ATOMIC exit occurs and the instruction is emulated with -an exclusive lock which ensures all emulation is serialised. - -While the atomic helpers look good enough for now there may be a need -to look at solutions that can more closely model the guest -architectures semantics. - -========== - -[1] https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/plain/Documentation/memory-barriers.txt diff --git a/docs/devel/tcg-icount.rst b/docs/devel/tcg-icount.rst new file mode 100644 index 0000000000..8d67b6c076 --- /dev/null +++ b/docs/devel/tcg-icount.rst @@ -0,0 +1,97 @@ +.. + Copyright (c) 2020, Linaro Limited + Written by Alex BennÃ©e + + +======================== +TCG Instruction Counting +======================== + +TCG has long supported a feature known as icount which allows for +instruction counting during execution. This should not be confused +with cycle accurate emulation - QEMU does not attempt to emulate how +long an instruction would take on real hardware. That is a job for +other more detailed (and slower) tools that simulate the rest of a +micro-architecture. + +This feature is only available for system emulation and is +incompatible with multi-threaded TCG. It can be used to better align +execution time with wall-clock time so a "slow" device doesn't run too +fast on modern hardware. It can also provides for a degree of +deterministic execution and is an essential part of the record/replay +support in QEMU. + +Core Concepts +============= + +At its heart icount is simply a count of executed instructions which +is stored in the TimersState of QEMU's timer sub-system. The number of +executed instructions can then be used to calculate QEMU_CLOCK_VIRTUAL +which represents the amount of elapsed time in the system since +execution started. Depending on the icount mode this may either be a +fixed number of ns per instruction or adjusted as execution continues +to keep wall clock time and virtual time in sync. + +To be able to calculate the number of executed instructions the +translator starts by allocating a budget of instructions to be +executed. The budget of instructions is limited by how long it will be +until the next timer will expire. We store this budget as part of a +vCPU icount_decr field which shared with the machinery for handling +cpu_exit(). The whole field is checked at the start of every +translated block and will cause a return to the outer loop to deal +with whatever caused the exit. + +In the case of icount, before the flag is checked we subtract the +number of instructions the translation block would execute. If this +would cause the instruction budget to go negative we exit the main +loop and regenerate a new translation block with exactly the right +number of instructions to take the budget to 0 meaning whatever timer +was due to expire will expire exactly when we exit the main run loop. + +Dealing with MMIO +----------------- + +While we can adjust the instruction budget for known events like timer +expiry we cannot do the same for MMIO. Every load/store we execute +might potentially trigger an I/O event, at which point we will need an +up to date and accurate reading of the icount number. + +To deal with this case, when an I/O access is made we: + + - restore un-executed instructions to the icount budget + - re-compile a single [1]_ instruction block for the current PC + - exit the cpu loop and execute the re-compiled block + +The new block is created with the CF_LAST_IO compile flag which +ensures the final instruction translation starts with a call to +gen_io_start() so we don't enter a perpetual loop constantly +recompiling a single instruction block. For translators using the +common translator_loop this is done automatically. + +.. [1] sometimes two instructions if dealing with delay slots + +Other I/O operations +-------------------- + +MMIO isn't the only type of operation for which we might need a +correct and accurate clock. IO port instructions and accesses to +system registers are the common examples here. These instructions have +to be handled by the individual translators which have the knowledge +of which operations are I/O operations. + +When the translator is handling an instruction of this kind: + +* it must call gen_io_start() if icount is enabled, at some + point before the generation of the code which actually does + the I/O, using a code fragment similar to: + +.. code:: c + + if (tb_cflags(s->base.tb) & CF_USE_ICOUNT) { + gen_io_start(); + } + +* it must end the TB immediately after this instruction + +Note that some older front-ends call a "gen_io_end()" function: +this is obsolete and should not be used. diff --git a/docs/system/gdb.rst b/docs/system/gdb.rst index a40145fcf8..abda961e2b 100644 --- a/docs/system/gdb.rst +++ b/docs/system/gdb.rst @@ -87,3 +87,23 @@ three commands you can query and set the single step behavior: (gdb) maintenance packet Qqemu.sstep=0x5 sending: "qemu.sstep=0x5" received: "OK" + + +Another feature that QEMU gdbstub provides is to toggle the memory GDB +works with, by default GDB will show the current process memory respecting +the virtual address translation. + +If you want to examine/change the physical memory you can set the gdbstub +to work with the physical memory rather with the virtual one. + +The memory mode can be checked by sending the following command: + +``maintenance packet qqemu.PhyMemMode`` + This will return either 0 or 1, 1 indicates you are currently in the + physical memory mode. + +``maintenance packet Qqemu.PhyMemMode:1`` + This will change the memory mode to physical memory. + +``maintenance packet Qqemu.PhyMemMode:0`` + This will change it back to normal memory mode. diff --git a/hw/display/Makefile.objs b/hw/display/Makefile.objs index e907f3182b..d619594ad4 100644 --- a/hw/display/Makefile.objs +++ b/hw/display/Makefile.objs @@ -49,19 +49,16 @@ common-obj-m += qxl.mo qxl.mo-objs = qxl.o qxl-logger.o qxl-render.o endif -ifeq ($(CONFIG_VIRTIO_GPU),y) -common-obj-m += virtio-gpu.mo -virtio-gpu-obj-$(CONFIG_VIRTIO_GPU) += virtio-gpu-base.o virtio-gpu.o virtio-gpu-3d.o -virtio-gpu-obj-$(CONFIG_VHOST_USER_GPU) += vhost-user-gpu.o -virtio-gpu-obj-$(call land,$(CONFIG_VIRTIO_GPU),$(CONFIG_VIRTIO_PCI)) += virtio-gpu-pci.o -virtio-gpu-obj-$(call land,$(CONFIG_VHOST_USER_GPU),$(CONFIG_VIRTIO_PCI)) += vhost-user-gpu-pci.o -virtio-gpu-obj-$(CONFIG_VIRTIO_VGA) += virtio-vga.o -virtio-gpu-obj-$(CONFIG_VHOST_USER_VGA) += vhost-user-vga.o -virtio-gpu.mo-objs := $(virtio-gpu-obj-y) -virtio-gpu.mo-cflags := $(VIRGL_CFLAGS) -virtio-gpu.mo-libs := $(VIRGL_LIBS) -endif - +common-obj-$(CONFIG_VIRTIO_GPU) += virtio-gpu-base.o virtio-gpu.o virtio-gpu-3d.o +common-obj-$(CONFIG_VHOST_USER_GPU) += vhost-user-gpu.o +common-obj-$(call land,$(CONFIG_VIRTIO_GPU),$(CONFIG_VIRTIO_PCI)) += virtio-gpu-pci.o +common-obj-$(call land,$(CONFIG_VHOST_USER_GPU),$(CONFIG_VIRTIO_PCI)) += vhost-user-gpu-pci.o +common-obj-$(CONFIG_VIRTIO_VGA) += virtio-vga.o +common-obj-$(CONFIG_VHOST_USER_VGA) += vhost-user-vga.o +virtio-gpu.o-cflags := $(VIRGL_CFLAGS) +virtio-gpu.o-libs += $(VIRGL_LIBS) +virtio-gpu-3d.o-cflags := $(VIRGL_CFLAGS) +virtio-gpu-3d.o-libs += $(VIRGL_LIBS) common-obj-$(CONFIG_DPCD) += dpcd.o common-obj-$(CONFIG_XLNX_ZYNQMP_ARM) += xlnx_dp.o diff --git a/linux-user/elfload.c b/linux-user/elfload.c index b5cb21384a..7e7f642332 100644 --- a/linux-user/elfload.c +++ b/linux-user/elfload.c @@ -2294,7 +2294,7 @@ static void pgb_dynamic(const char *image_name, long align) static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr, abi_ulong guest_hiaddr, long align) { - const int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE; + int flags = MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE; void *addr, *test; if (guest_hiaddr > reserved_va) { @@ -2307,15 +2307,19 @@ static void pgb_reserved_va(const char *image_name, abi_ulong guest_loaddr, /* Widen the "image" to the entire reserved address space. */ pgb_static(image_name, 0, reserved_va, align); +#ifdef MAP_FIXED_NOREPLACE + flags |= MAP_FIXED_NOREPLACE; +#endif + /* Reserve the memory on the host. */ assert(guest_base != 0); test = g2h(0); addr = mmap(test, reserved_va, PROT_NONE, flags, -1, 0); if (addr == MAP_FAILED) { error_report("Unable to reserve 0x%lx bytes of virtual address " - "space for use as guest address space (check your " + "space (%s) for use as guest address space (check your " "virtual memory ulimit setting or reserve less " - "using -R option)", reserved_va); + "using -R option)", reserved_va, strerror(errno)); exit(EXIT_FAILURE); } assert(addr == test); diff --git a/python/qemu/console_socket.py b/python/qemu/console_socket.py new file mode 100644 index 0000000000..830cb7c628 --- /dev/null +++ b/python/qemu/console_socket.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# +# This python module implements a ConsoleSocket object which is +# designed always drain the socket itself, and place +# the bytes into a in memory buffer for later processing. +# +# Optionally a file path can be passed in and we will also +# dump the characters to this file for debug. +# +# Copyright 2020 Linaro +# +# Authors: +# Robert Foley <robert.foley@xxxxxxxxxx> +# +# This code is licensed under the GPL version 2 or later. See +# the COPYING file in the top-level directory. +# +import asyncore +import socket +import threading +import io +import os +import sys +from collections import deque +import time +import traceback + +class ConsoleSocket(asyncore.dispatcher): + + def __init__(self, address, file=None): + self._recv_timeout_sec = 300 + self._buffer = deque() + self._asyncore_thread = None + self._sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._sock.connect(address) + self._logfile = None + if file: + self._logfile = open(file, "w") + asyncore.dispatcher.__init__(self, sock=self._sock) + self._open = True + self._thread_start() + + def _thread_start(self): + """Kick off a thread to wait on the asyncore.loop""" + if self._asyncore_thread is not None: + return + self._asyncore_thread = threading.Thread(target=asyncore.loop, + kwargs={'timeout':1}) + self._asyncore_thread.daemon = True + self._asyncore_thread.start() + + def handle_close(self): + """redirect close to base class""" + # Call the base class close, but not self.close() since + # handle_close() occurs in the context of the thread which + # self.close() attempts to join. + asyncore.dispatcher.close(self) + + def close(self): + """Close the base object and wait for the thread to terminate""" + if self._open: + self._open = False + asyncore.dispatcher.close(self) + if self._asyncore_thread is not None: + thread, self._asyncore_thread = self._asyncore_thread, None + thread.join() + if self._logfile: + self._logfile.close() + self._logfile = None + + def handle_read(self): + """process arriving characters into in memory _buffer""" + try: + data = asyncore.dispatcher.recv(self, 1) + # latin1 is needed since there are some chars + # we are receiving that cannot be encoded to utf-8 + # such as 0xe2, 0x80, 0xA6. + string = data.decode("latin1") + except: + print("Exception seen.") + traceback.print_exc() + return + if self._logfile: + self._logfile.write("{}".format(string)) + self._logfile.flush() + for c in string: + self._buffer.extend(c) + + def recv(self, n=1, sleep_delay_s=0.1): + """Return chars from in memory buffer""" + start_time = time.time() + while len(self._buffer) < n: + time.sleep(sleep_delay_s) + elapsed_sec = time.time() - start_time + if elapsed_sec > self._recv_timeout_sec: + raise socket.timeout + chars = ''.join([self._buffer.popleft() for i in range(n)]) + # We choose to use latin1 to remain consistent with + # handle_read() and give back the same data as the user would + # receive if they were reading directly from the + # socket w/o our intervention. + return chars.encode("latin1") + + def set_blocking(self): + """Maintain compatibility with socket API""" + pass + + def settimeout(self, seconds): + """Set current timeout on recv""" + self._recv_timeout_sec = seconds diff --git a/python/qemu/machine.py b/python/qemu/machine.py index 041c615052..c25f0b42cf 100644 --- a/python/qemu/machine.py +++ b/python/qemu/machine.py @@ -26,6 +26,7 @@ import socket import tempfile from typing import Optional, Type from types import TracebackType +from qemu.console_socket import ConsoleSocket from . import qmp @@ -75,7 +76,8 @@ class QEMUMachine: def __init__(self, binary, args=None, wrapper=None, name=None, test_dir="/var/tmp", monitor_address=None, - socket_scm_helper=None, sock_dir=None): + socket_scm_helper=None, sock_dir=None, + drain_console=False, console_log=None): ''' Initialize a QEMUMachine @@ -86,6 +88,9 @@ class QEMUMachine: @param test_dir: where to create socket and log file @param monitor_address: address for QMP monitor @param socket_scm_helper: helper program, required for send_fd_scm() + @param sock_dir: where to create socket (overrides test_dir for sock) + @param console_log: (optional) path to console log file + @param drain_console: (optional) True to drain console socket to buffer @note: Qemu process is not started until launch() is used. ''' if args is None: @@ -122,6 +127,12 @@ class QEMUMachine: self._console_address = None self._console_socket = None self._remove_files = [] + self._console_log_path = console_log + if self._console_log_path: + # In order to log the console, buffering needs to be enabled. + self._drain_console = True + else: + self._drain_console = drain_console def __enter__(self): return self @@ -580,7 +591,11 @@ class QEMUMachine: Returns a socket connected to the console """ if self._console_socket is None: - self._console_socket = socket.socket(socket.AF_UNIX, - socket.SOCK_STREAM) - self._console_socket.connect(self._console_address) + if self._drain_console: + self._console_socket = ConsoleSocket(self._console_address, + file=self._console_log_path) + else: + self._console_socket = socket.socket(socket.AF_UNIX, + socket.SOCK_STREAM) + self._console_socket.connect(self._console_address) return self._console_socket diff --git a/tests/Makefile.include b/tests/Makefile.include index 09df2d3f86..c7e4646ded 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -22,6 +22,8 @@ endif @echo " $(MAKE) check-venv Creates a Python venv for tests" @echo " $(MAKE) check-clean Clean the tests and related data" @echo + @echo "The following are useful for CI builds" + @echo " $(MAKE) check-build Build most test binaris" @echo " $(MAKE) get-vm-images Downloads all images used by acceptance tests, according to configured targets (~350 MB each, 1.5 GB max)" @echo @echo @@ -649,6 +651,10 @@ $(patsubst %, check-qtest-%, $(QTEST_TARGETS)): check-qtest-%: %-softmmu/all $(c QTEST_QEMU_BINARY=$*-softmmu/qemu-system-$* \ QTEST_QEMU_IMG=qemu-img$(EXESUF)) +build-qtest: $(patsubst %, %-softmmu/all, $(QTEST_TARGETS)) $(check-qtest-y) + +build-unit: $(check-unit-y) + check-unit: $(check-unit-y) $(call do_test_human, $^) @@ -680,7 +686,6 @@ check-report.tap: $(patsubst %,check-report-qtest-%.tap, $(QTEST_TARGETS)) check FP_TEST_BIN=$(BUILD_DIR)/tests/fp/fp-test # the build dir is created by configure -.PHONY: $(FP_TEST_BIN) $(FP_TEST_BIN): config-host.h $(test-util-obj-y) $(call quiet-command, \ $(MAKE) $(SUBDIR_MAKEFLAGS) -C $(dir $@) V="$(V)" $(notdir $@), \ @@ -814,9 +819,10 @@ check-softfloat-ops: $(SF_MATH_RULES) .PHONY: check-softfloat ifeq ($(CONFIG_TCG),y) -check-softfloat: check-softfloat-conv check-softfloat-compare check-softfloat-ops +build-softfloat: $(FP_TEST_BIN) +check-softfloat: build-softfloat check-softfloat-conv check-softfloat-compare check-softfloat-ops else -check-softfloat: +build-softfloat check-softfloat: $(call quiet-command, /bin/true, "FLOAT TEST", \ "SKIPPED for non-TCG builds") endif @@ -944,7 +950,7 @@ check-acceptance: check-venv $(TESTS_RESULTS_DIR) get-vm-images --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ --filter-by-tags-include-empty --filter-by-tags-include-empty-key \ $(AVOCADO_TAGS) \ - --failfast=on tests/acceptance, \ + $(if $(GITLAB_CI),,--failfast=on) tests/acceptance, \ "AVOCADO", "tests/acceptance") # Consolidated targets @@ -955,7 +961,8 @@ check-qtest: $(patsubst %,check-qtest-%, $(QTEST_TARGETS)) ifeq ($(CONFIG_TOOLS),y) check-block: $(patsubst %,check-%, $(check-block-y)) endif -check: check-block check-qapi-schema check-unit check-softfloat check-qtest check-decodetree +check-build: build-unit build-softfloat build-qtest + check-clean: rm -rf $(check-unit-y) tests/*.o tests/*/*.o $(QEMU_IOTESTS_HELPERS-y) rm -rf $(sort $(foreach target,$(SYSEMU_TARGET_LIST), $(check-qtest-$(target)-y:%=tests/qtest/%$(EXESUF))) $(check-qtest-generic-y:%=tests/qtest/%$(EXESUF))) @@ -963,6 +970,8 @@ check-clean: rm -f tests/qtest/dbus-vmstate1-gen-timestamp rm -rf $(TESTS_VENV_DIR) $(TESTS_RESULTS_DIR) +check: check-block check-qapi-schema check-unit check-softfloat check-qtest check-decodetree + clean: check-clean # Build the help program automatically diff --git a/tests/acceptance/boot_linux.py b/tests/acceptance/boot_linux.py index 3aa57e88b0..0055dc7cee 100644 --- a/tests/acceptance/boot_linux.py +++ b/tests/acceptance/boot_linux.py @@ -20,6 +20,7 @@ from avocado.utils import network from avocado.utils import vmimage from avocado.utils import datadrainer from avocado.utils.path import find_command +from avocado import skipIf ACCEL_NOT_AVAILABLE_FMT = "%s accelerator does not seem to be available" KVM_NOT_AVAILABLE = ACCEL_NOT_AVAILABLE_FMT % "KVM" @@ -220,6 +221,7 @@ class BootLinuxS390X(BootLinux): chksum = '4caaab5a434fd4d1079149a072fdc7891e354f834d355069ca982fdcaf5a122d' + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_s390_ccw_virtio_tcg(self): """ :avocado: tags=machine:s390-ccw-virtio diff --git a/tests/acceptance/linux_initrd.py b/tests/acceptance/linux_initrd.py index a3e54d3fc9..a249e2f14a 100644 --- a/tests/acceptance/linux_initrd.py +++ b/tests/acceptance/linux_initrd.py @@ -8,10 +8,12 @@ # This work is licensed under the terms of the GNU GPL, version 2 or # later. See the COPYING file in the top-level directory. +import os import logging import tempfile from avocado_qemu import Test +from avocado import skipIf class LinuxInitrd(Test): @@ -51,6 +53,7 @@ class LinuxInitrd(Test): max_size + 1) self.assertRegex(self.vm.get_log(), expected_msg) + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_with_2gib_file_should_work_with_linux_v4_16(self): """ QEMU has supported up to 4 GiB initrd for recent kernel diff --git a/tests/acceptance/machine_mips_malta.py b/tests/acceptance/machine_mips_malta.py index 92b4f28a11..7c9a4ee4d2 100644 --- a/tests/acceptance/machine_mips_malta.py +++ b/tests/acceptance/machine_mips_malta.py @@ -15,6 +15,7 @@ from avocado import skipUnless from avocado_qemu import Test from avocado_qemu import wait_for_console_pattern from avocado.utils import archive +from avocado import skipIf NUMPY_AVAILABLE = True @@ -99,6 +100,7 @@ class MaltaMachineFramebuffer(Test): """ self.do_test_i6400_framebuffer_logo(1) + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_7cores(self): """ :avocado: tags=arch:mips64el @@ -108,6 +110,7 @@ class MaltaMachineFramebuffer(Test): """ self.do_test_i6400_framebuffer_logo(7) + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_mips_malta_i6400_framebuffer_logo_8cores(self): """ :avocado: tags=arch:mips64el diff --git a/tests/acceptance/machine_rx_gdbsim.py b/tests/acceptance/machine_rx_gdbsim.py index a44f2c87da..bff63e421d 100644 --- a/tests/acceptance/machine_rx_gdbsim.py +++ b/tests/acceptance/machine_rx_gdbsim.py @@ -50,7 +50,7 @@ class RxGdbSimMachine(Test): :avocado: tags=machine:gdbsim-r5f562n7 :avocado: tags=endian:little """ - dtb_url = ('https://acc.dl.osdn.jp/users/23/23887/rx-qemu.dtb') + dtb_url = ('https://acc.dl.osdn.jp/users/23/23887/rx-virt.dtb') dtb_hash = '7b4e4e2c71905da44e86ce47adee2210b026ac18' dtb_path = self.fetch_asset(dtb_url, asset_hash=dtb_hash) kernel_url = ('http://acc.dl.osdn.jp/users/23/23845/zImage') diff --git a/tests/acceptance/replay_kernel.py b/tests/acceptance/replay_kernel.py index 60621417dd..62d2db8c64 100644 --- a/tests/acceptance/replay_kernel.py +++ b/tests/acceptance/replay_kernel.py @@ -73,7 +73,7 @@ class ReplayKernel(LinuxKernelTest): logger = logging.getLogger('replay') logger.info('replay overhead {:.2%}'.format(t2 / t1 - 1)) - @skipIf(os.getenv('CONTINUOUS_INTEGRATION'), 'Running on Travis-CI') + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_x86_64_pc(self): """ :avocado: tags=arch:x86_64 diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 3e3617816e..a104e9df28 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -13,6 +13,7 @@ DOCKER_IMAGES := $(sort $(notdir $(basename $(wildcard $(DOCKER_FILES_DIR)/*.doc DOCKER_TARGETS := $(patsubst %,docker-image-%,$(DOCKER_IMAGES)) # Use a global constant ccache directory to speed up repetitive builds DOCKER_CCACHE_DIR := $$HOME/.cache/qemu-docker-ccache +DOCKER_REGISTRY := $(if $(REGISTRY),$(REGISTRY),registry.gitlab.com/qemu-project/qemu) DOCKER_TESTS := $(notdir $(shell \ find $(SRC_PATH)/tests/docker/ -name 'test-*' -type f)) @@ -50,13 +51,15 @@ docker-image: ${DOCKER_TARGETS} ifdef SKIP_DOCKER_BUILD docker-image-%: $(DOCKER_FILES_DIR)/%.docker $(call quiet-command, \ - $(DOCKER_SCRIPT) check --quiet qemu:$* $<, \ + $(DOCKER_SCRIPT) check --quiet qemu/$* $<, \ "CHECK", "$*") else docker-image-%: $(DOCKER_FILES_DIR)/%.docker $(call quiet-command,\ - $(DOCKER_SCRIPT) build -t qemu:$* -f $< \ - $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ + $(DOCKER_SCRIPT) build -t qemu/$* -f $< \ + $(if $V,,--quiet) \ + $(if $(NOCACHE),--no-cache, \ + $(if $(DOCKER_REGISTRY),--registry $(DOCKER_REGISTRY))) \ $(if $(NOUSER),,--add-current-user) \ $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES))\ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)),\ @@ -75,14 +78,14 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker DEB_ARCH=$(DEB_ARCH) \ DEB_TYPE=$(DEB_TYPE) \ $(if $(DEB_URL),DEB_URL=$(DEB_URL),) \ - $(DOCKER_SCRIPT) build qemu:debian-$* $< \ + $(DOCKER_SCRIPT) build qemu/debian-$* $< \ $(if $V,,--quiet) $(if $(NOCACHE),--no-cache) \ $(if $(NOUSER),,--add-current-user) \ $(if $(EXTRA_FILES),--extra-files $(EXTRA_FILES)) \ $(if $(EXECUTABLE),--include-executable=$(EXECUTABLE)), \ "BUILD","binfmt debian-$* (debootstrapped)"), \ $(call quiet-command, \ - $(DOCKER_SCRIPT) check --quiet qemu:debian-$* $< || \ + $(DOCKER_SCRIPT) check --quiet qemu/debian-$* $< || \ { echo "You will need to build $(EXECUTABLE)"; exit 1;},\ "CHECK", "debian-$* exists")) @@ -131,6 +134,7 @@ docker-image-travis: NOUSER=1 # Specialist build images, sometimes very limited tools docker-image-debian-tricore-cross: docker-image-debian9 +docker-image-debian-all-test-cross: docker-image-debian10 docker-image-debian-arm64-test-cross: docker-image-debian11 # These images may be good enough for building tests but not for test builds @@ -213,6 +217,7 @@ endif @echo ' Include extra files in image.' @echo ' ENGINE=auto/docker/podman' @echo ' Specify which container engine to run.' + @echo ' REGISTRY=url Cache builds from registry (default:$(DOCKER_REGISTRY))' # This rule if for directly running against an arbitrary docker target. # It is called by the expanded docker targets (e.g. make @@ -258,7 +263,7 @@ docker-run: docker-qemu-src docker-run-%: CMD = $(shell echo '$@' | sed -e 's/docker-run-$[^@]*$@$.*$/\1/') docker-run-%: IMAGE = $(shell echo '$@' | sed -e 's/docker-run-$[^@]*$@$.*$/\2/') docker-run-%: - @$(MAKE) docker-run TEST=$(CMD) IMAGE=qemu:$(IMAGE) + @$(MAKE) docker-run TEST=$(CMD) IMAGE=qemu/$(IMAGE) docker-clean: $(call quiet-command, $(DOCKER_SCRIPT) clean) diff --git a/tests/docker/common.rc b/tests/docker/common.rc index 02cd67a8c5..ebc5b97ecf 100755 --- a/tests/docker/common.rc +++ b/tests/docker/common.rc @@ -47,7 +47,7 @@ build_qemu() check_qemu() { # default to make check unless the caller specifies - if test -z "$@"; then + if [ $# = 0 ]; then INVOCATION="check" else INVOCATION="$@" diff --git a/tests/docker/docker.py b/tests/docker/docker.py index e630aae108..2d67bbd15a 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -204,7 +204,7 @@ def _dockerfile_preprocess(df): for l in df.splitlines(): if len(l.strip()) == 0 or l.startswith("#"): continue - from_pref = "FROM qemu:" + from_pref = "FROM qemu/" if l.startswith(from_pref): # TODO: Alternatively we could replace this line with "FROM $ID" # where $ID is the image's hex id obtained with @@ -221,6 +221,13 @@ class Docker(object): """ Running Docker commands """ def __init__(self): self._command = _guess_engine_command() + + if "docker" in self._command and "TRAVIS" not in os.environ: + os.environ["DOCKER_BUILDKIT"] = "1" + self._buildkit = True + else: + self._buildkit = False + self._instance = None atexit.register(self._kill_instances) signal.signal(signal.SIGTERM, self._kill_instances) @@ -289,10 +296,25 @@ class Docker(object): return labels.get("com.qemu.dockerfile-checksum", "") def build_image(self, tag, docker_dir, dockerfile, - quiet=True, user=False, argv=None, extra_files_cksum=[]): + quiet=True, user=False, argv=None, registry=None, + extra_files_cksum=[]): if argv is None: argv = [] + # pre-calculate the docker checksum before any + # substitutions we make for caching + checksum = _text_checksum(_dockerfile_preprocess(dockerfile)) + + if registry is not None: + # see if we can fetch a cache copy, may fail... + pull_args = ["pull", "%s/%s" % (registry, tag)] + if self._do(pull_args, quiet=quiet) == 0: + dockerfile = dockerfile.replace("FROM qemu/", + "FROM %s/qemu/" % + (registry)) + else: + registry = None + tmp_df = tempfile.NamedTemporaryFile(mode="w+t", encoding='utf-8', dir=docker_dir, suffix=".docker") @@ -306,15 +328,23 @@ class Docker(object): (uname, uid, uname)) tmp_df.write("\n") - tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % - _text_checksum(_dockerfile_preprocess(dockerfile))) + tmp_df.write("LABEL com.qemu.dockerfile-checksum=%s" % (checksum)) for f, c in extra_files_cksum: tmp_df.write("LABEL com.qemu.%s-checksum=%s" % (f, c)) tmp_df.flush() - self._do_check(["build", "-t", tag, "-f", tmp_df.name] + argv + - [docker_dir], + build_args = ["build", "-t", tag, "-f", tmp_df.name] + if self._buildkit: + build_args += ["--build-arg", "BUILDKIT_INLINE_CACHE=1"] + + if registry is not None: + cache = "%s/%s" % (registry, tag) + build_args += ["--cache-from", cache] + build_args += argv + build_args += [docker_dir] + + self._do_check(build_args, quiet=quiet) def update_image(self, tag, tarball, quiet=True): @@ -403,6 +433,8 @@ class BuildCommand(SubCommand): parser.add_argument("--add-current-user", "-u", dest="user", action="store_true", help="Add the current user to image's passwd") + parser.add_argument("--registry", "-r", + help="cache from docker registry") parser.add_argument("-t", dest="tag", help="Image Tag") parser.add_argument("-f", dest="dockerfile", @@ -458,7 +490,8 @@ class BuildCommand(SubCommand): for k, v in os.environ.items() if k.lower() in FILTERED_ENV_NAMES] dkr.build_image(tag, docker_dir, dockerfile, - quiet=args.quiet, user=args.user, argv=argv, + quiet=args.quiet, user=args.user, + argv=argv, registry=args.registry, extra_files_cksum=cksum) rmtree(docker_dir) diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker new file mode 100644 index 0000000000..dedcea58b4 --- /dev/null +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -0,0 +1,53 @@ +# +# Docker all cross-compiler target (tests only) +# +# While the normal cross builds take care to setup proper multiarch +# build environments which can cross build QEMU this just installs the +# basic compilers for as many targets as possible. We shall use this +# to build and run linux-user tests on GitLab +# +FROM qemu/debian10 + +# What we need to build QEMU itself +RUN apt update && \ + DEBIAN_FRONTEND=noninteractive eatmydata \ + apt build-dep -yy qemu + +# Add the foreign architecture we want and install dependencies +RUN DEBIAN_FRONTEND=noninteractive eatmydata \ + apt install -y --no-install-recommends \ + gcc-aarch64-linux-gnu \ + libc6-dev-arm64-cross \ + gcc-alpha-linux-gnu \ + libc6.1-dev-alpha-cross \ + gcc-arm-linux-gnueabihf \ + libc6-dev-armhf-cross \ + gcc-hppa-linux-gnu \ + libc6-dev-hppa-cross \ + gcc-m68k-linux-gnu \ + libc6-dev-m68k-cross \ + gcc-mips-linux-gnu \ + libc6-dev-mips-cross \ + gcc-mips64-linux-gnuabi64 \ + libc6-dev-mips64-cross \ + gcc-mips64el-linux-gnuabi64 \ + libc6-dev-mips64el-cross \ + gcc-mipsel-linux-gnu \ + libc6-dev-mipsel-cross \ + gcc-powerpc-linux-gnu \ + libc6-dev-powerpc-cross \ + gcc-powerpc64-linux-gnu \ + libc6-dev-ppc64-cross \ + gcc-powerpc64le-linux-gnu \ + libc6-dev-ppc64el-cross \ + gcc-riscv64-linux-gnu \ + libc6-dev-riscv64-cross \ + gcc-s390x-linux-gnu \ + libc6-dev-s390x-cross \ + gcc-sh4-linux-gnu \ + libc6-dev-sh4-cross \ + gcc-sparc64-linux-gnu \ + libc6-dev-sparc64-cross + +ENV QEMU_CONFIGURE_OPTS --disable-system --disable-docs --disable-tools +ENV DEF_TARGET_LIST aarch64-linux-user,alpha-linux-user,arm-linux-user,hppa-linux-user,i386-linux-user,m68k-linux-user,mips-linux-user,mips64-linux-user,mips64el-linux-user,mipsel-linux-user,ppc-linux-user,ppc64-linux-user,ppc64le-linux-user,riscv64-linux-user,s390x-linux-user,sh4-linux-user,sparc64-linux-user diff --git a/tests/docker/dockerfiles/debian-alpha-cross.docker b/tests/docker/dockerfiles/debian-alpha-cross.docker index 74bcabfdb1..10fe30df0d 100644 --- a/tests/docker/dockerfiles/debian-alpha-cross.docker +++ b/tests/docker/dockerfiles/debian-alpha-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 5d89041925..870109ef6a 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -4,7 +4,7 @@ # This docker target is used on non-x86_64 machines which need the # x86_64 cross compilers installed. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Alex BennÃ©e <alex.bennee@xxxxxxxxxx> # Add the foreign architecture we want and install dependencies diff --git a/tests/docker/dockerfiles/debian-amd64.docker b/tests/docker/dockerfiles/debian-amd64.docker index 957f0bc2e7..8fdfd6a6b0 100644 --- a/tests/docker/dockerfiles/debian-amd64.docker +++ b/tests/docker/dockerfiles/debian-amd64.docker @@ -4,7 +4,7 @@ # This docker target builds on the debian Stretch base image. Further # libraries which are not widely available are installed by hand. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> RUN apt update && \ diff --git a/tests/docker/dockerfiles/debian-arm64-cross.docker b/tests/docker/dockerfiles/debian-arm64-cross.docker index 09ca0a1ba7..166e24df13 100644 --- a/tests/docker/dockerfiles/debian-arm64-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 # Add the foreign architecture we want and install dependencies RUN dpkg --add-architecture arm64 diff --git a/tests/docker/dockerfiles/debian-arm64-test-cross.docker b/tests/docker/dockerfiles/debian-arm64-test-cross.docker index a44e76d942..53a9012beb 100644 --- a/tests/docker/dockerfiles/debian-arm64-test-cross.docker +++ b/tests/docker/dockerfiles/debian-arm64-test-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Bullseye base image. # -FROM qemu:debian11 +FROM qemu/debian11 # Add the foreign architecture we want and install dependencies RUN dpkg --add-architecture arm64 diff --git a/tests/docker/dockerfiles/debian-armel-cross.docker b/tests/docker/dockerfiles/debian-armel-cross.docker index e3794a61c9..b7b1a3585f 100644 --- a/tests/docker/dockerfiles/debian-armel-cross.docker +++ b/tests/docker/dockerfiles/debian-armel-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> # Add the foreign architecture we want and install dependencies diff --git a/tests/docker/dockerfiles/debian-armhf-cross.docker b/tests/docker/dockerfiles/debian-armhf-cross.docker index e163b8b956..25d7618833 100644 --- a/tests/docker/dockerfiles/debian-armhf-cross.docker +++ b/tests/docker/dockerfiles/debian-armhf-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 # Add the foreign architecture we want and install dependencies RUN dpkg --add-architecture armhf diff --git a/tests/docker/dockerfiles/debian-hppa-cross.docker b/tests/docker/dockerfiles/debian-hppa-cross.docker index 5c68b2d330..3d6c65a3ef 100644 --- a/tests/docker/dockerfiles/debian-hppa-cross.docker +++ b/tests/docker/dockerfiles/debian-hppa-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-m68k-cross.docker b/tests/docker/dockerfiles/debian-m68k-cross.docker index 25edc80e9a..fcb10e3534 100644 --- a/tests/docker/dockerfiles/debian-m68k-cross.docker +++ b/tests/docker/dockerfiles/debian-m68k-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-mips-cross.docker b/tests/docker/dockerfiles/debian-mips-cross.docker index 08a8e1c29c..26c154014d 100644 --- a/tests/docker/dockerfiles/debian-mips-cross.docker +++ b/tests/docker/dockerfiles/debian-mips-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-mips64-cross.docker b/tests/docker/dockerfiles/debian-mips64-cross.docker index 1a79505d69..09c2ba584e 100644 --- a/tests/docker/dockerfiles/debian-mips64-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-mips64el-cross.docker b/tests/docker/dockerfiles/debian-mips64el-cross.docker index 453b53ef72..c990b683b7 100644 --- a/tests/docker/dockerfiles/debian-mips64el-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64el-cross.docker @@ -4,7 +4,7 @@ # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-mipsel-cross.docker b/tests/docker/dockerfiles/debian-mipsel-cross.docker index 3b6e975c68..0e5dd42d3c 100644 --- a/tests/docker/dockerfiles/debian-mipsel-cross.docker +++ b/tests/docker/dockerfiles/debian-mipsel-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-powerpc-cross.docker b/tests/docker/dockerfiles/debian-powerpc-cross.docker index 89dd4fbf87..07e1789650 100644 --- a/tests/docker/dockerfiles/debian-powerpc-cross.docker +++ b/tests/docker/dockerfiles/debian-powerpc-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-ppc64-cross.docker b/tests/docker/dockerfiles/debian-ppc64-cross.docker index 4bf88ab02d..8efe68874e 100644 --- a/tests/docker/dockerfiles/debian-ppc64-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64-cross.docker @@ -2,7 +2,7 @@ # Docker ppc64 cross-compiler target # # This docker target builds on the debian Buster base image. -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-ppc64el-cross.docker b/tests/docker/dockerfiles/debian-ppc64el-cross.docker index cd386f01d9..1146a06be6 100644 --- a/tests/docker/dockerfiles/debian-ppc64el-cross.docker +++ b/tests/docker/dockerfiles/debian-ppc64el-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 # Add the foreign architecture we want and install dependencies RUN dpkg --add-architecture ppc64el && \ diff --git a/tests/docker/dockerfiles/debian-riscv64-cross.docker b/tests/docker/dockerfiles/debian-riscv64-cross.docker index 5e2d6ddb60..2bbff19772 100644 --- a/tests/docker/dockerfiles/debian-riscv64-cross.docker +++ b/tests/docker/dockerfiles/debian-riscv64-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-s390x-cross.docker b/tests/docker/dockerfiles/debian-s390x-cross.docker index 43fe59836f..9f2ab51eb0 100644 --- a/tests/docker/dockerfiles/debian-s390x-cross.docker +++ b/tests/docker/dockerfiles/debian-s390x-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian10 +FROM qemu/debian10 # Add the s390x architecture RUN dpkg --add-architecture s390x diff --git a/tests/docker/dockerfiles/debian-sh4-cross.docker b/tests/docker/dockerfiles/debian-sh4-cross.docker index 9d7663764e..fd3af89575 100644 --- a/tests/docker/dockerfiles/debian-sh4-cross.docker +++ b/tests/docker/dockerfiles/debian-sh4-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-sparc64-cross.docker b/tests/docker/dockerfiles/debian-sparc64-cross.docker index 31fd34f120..f4bb9b561c 100644 --- a/tests/docker/dockerfiles/debian-sparc64-cross.docker +++ b/tests/docker/dockerfiles/debian-sparc64-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Buster base image. # -FROM qemu:debian10 +FROM qemu/debian10 RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ diff --git a/tests/docker/dockerfiles/debian-tricore-cross.docker b/tests/docker/dockerfiles/debian-tricore-cross.docker index 4a0f7706a3..769d95c77b 100644 --- a/tests/docker/dockerfiles/debian-tricore-cross.docker +++ b/tests/docker/dockerfiles/debian-tricore-cross.docker @@ -7,7 +7,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later # -FROM qemu:debian9 +FROM qemu/debian9 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-win32-cross.docker b/tests/docker/dockerfiles/debian-win32-cross.docker index d16d6431bc..b045e821b9 100644 --- a/tests/docker/dockerfiles/debian-win32-cross.docker +++ b/tests/docker/dockerfiles/debian-win32-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch MXE base image. # -FROM qemu:debian9-mxe +FROM qemu/debian9-mxe MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-win64-cross.docker b/tests/docker/dockerfiles/debian-win64-cross.docker index b0bc960445..2fc9cfcbc6 100644 --- a/tests/docker/dockerfiles/debian-win64-cross.docker +++ b/tests/docker/dockerfiles/debian-win64-cross.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch MXE base image. # -FROM qemu:debian9-mxe +FROM qemu/debian9-mxe MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/debian-xtensa-cross.docker b/tests/docker/dockerfiles/debian-xtensa-cross.docker index beb73f46ba..ba4148299c 100644 --- a/tests/docker/dockerfiles/debian-xtensa-cross.docker +++ b/tests/docker/dockerfiles/debian-xtensa-cross.docker @@ -18,12 +18,12 @@ RUN apt-get update && \ git \ python3-minimal -ENV CPU_LIST csp dc232b dc233c -ENV TOOLCHAIN_RELEASE 2018.02 +ENV CPU_LIST dc232b dc233c de233_fpu dsp3400 +ENV TOOLCHAIN_RELEASE 2020.07 RUN for cpu in $CPU_LIST; do \ curl -#SL http://github.com/foss-xtensa/toolchain/releases/download/$TOOLCHAIN_RELEASE/x86_64-$TOOLCHAIN_RELEASE-xtensa-$cpu-elf.tar.gz \ | tar -xzC /opt; \ done -ENV PATH $PATH:/opt/$TOOLCHAIN_RELEASE/xtensa-dc232b-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-dc233c-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-csp-elf/bin +ENV PATH $PATH:/opt/$TOOLCHAIN_RELEASE/xtensa-dc232b-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-dc233c-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-de233_fpu-elf/bin:/opt/$TOOLCHAIN_RELEASE/xtensa-dsp3400-elf/bin diff --git a/tests/docker/dockerfiles/debian9-mxe.docker b/tests/docker/dockerfiles/debian9-mxe.docker index 62ff1cecf2..ae2c222a6f 100644 --- a/tests/docker/dockerfiles/debian9-mxe.docker +++ b/tests/docker/dockerfiles/debian9-mxe.docker @@ -3,7 +3,7 @@ # # This docker target builds on the debian Stretch base image. # -FROM qemu:debian9 +FROM qemu/debian9 MAINTAINER Philippe Mathieu-DaudÃ© <f4bug@xxxxxxxxx> diff --git a/tests/docker/dockerfiles/fedora.docker b/tests/docker/dockerfiles/fedora.docker index 798ddd2c3e..70b6186bd3 100644 --- a/tests/docker/dockerfiles/fedora.docker +++ b/tests/docker/dockerfiles/fedora.docker @@ -80,7 +80,12 @@ ENV PACKAGES \ pixman-devel \ python3 \ python3-PyYAML \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ python3-sphinx \ + python3-virtualenv \ rdma-core-devel \ SDL2-devel \ snappy-devel \ @@ -89,6 +94,8 @@ ENV PACKAGES \ systemd-devel \ systemtap-sdt-devel \ tar \ + tesseract \ + tesseract-langpack-eng \ texinfo \ usbredir-devel \ virglrenderer-devel \ diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 6050ce7e8a..f7aac840bf 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -46,9 +46,17 @@ ENV PACKAGES flex bison \ libxen-dev \ libzstd-dev \ make \ - python3-yaml \ + python3-numpy \ + python3-opencv \ + python3-pil \ + python3-pip \ python3-sphinx \ + python3-venv \ + python3-yaml \ + rpm2cpio \ sparse \ + tesseract-ocr \ + tesseract-ocr-eng \ texinfo \ xfslibs-dev\ vim diff --git a/tests/qemu-iotests/common.filter b/tests/qemu-iotests/common.filter index d967adc59a..c9f978abce 100644 --- a/tests/qemu-iotests/common.filter +++ b/tests/qemu-iotests/common.filter @@ -186,7 +186,7 @@ _filter_img_create() -e 's/^$data_file$/3-\1/' \ -e 's/^$encryption$/4-\1/' \ -e 's/^$preallocation$/8-\1/' \ - | sort \ + | LC_ALL=C sort \ | $SED -e 's/^[0-9]-//' \ | tr '\n\0' ' \n' \ | $SED -e 's/^ *$//' -e 's/ *$//' diff --git a/tests/qht-bench.c b/tests/qht-bench.c index eb88a90137..362f03cb03 100644 --- a/tests/qht-bench.c +++ b/tests/qht-bench.c @@ -25,7 +25,13 @@ struct thread_stats { struct thread_info { void (*func)(struct thread_info *); struct thread_stats stats; - uint64_t r; + /* + * Seed is in the range [1..UINT64_MAX], because the RNG requires + * a non-zero seed. To use, subtract 1 and compare against the + * threshold with </>=. This lets threshold = 0 never match (0% hit), + * and threshold = UINT64_MAX always match (100% hit). + */ + uint64_t seed; bool write_op; /* writes alternate between insertions and removals */ bool resize_down; } QEMU_ALIGNED(64); /* avoid false sharing among threads */ @@ -131,8 +137,9 @@ static uint64_t xorshift64star(uint64_t x) static void do_rz(struct thread_info *info) { struct thread_stats *stats = &info->stats; + uint64_t r = info->seed - 1; - if (info->r < resize_threshold) { + if (r < resize_threshold) { size_t size = info->resize_down ? resize_min : resize_max; bool resized; @@ -151,13 +158,14 @@ static void do_rz(struct thread_info *info) static void do_rw(struct thread_info *info) { struct thread_stats *stats = &info->stats; + uint64_t r = info->seed - 1; uint32_t hash; long *p; - if (info->r >= update_threshold) { + if (r >= update_threshold) { bool read; - p = &keys[info->r & (lookup_range - 1)]; + p = &keys[r & (lookup_range - 1)]; hash = hfunc(*p); read = qht_lookup(&ht, p, hash); if (read) { @@ -166,7 +174,7 @@ static void do_rw(struct thread_info *info) stats->not_rd++; } } else { - p = &keys[info->r & (update_range - 1)]; + p = &keys[r & (update_range - 1)]; hash = hfunc(*p); if (info->write_op) { bool written = false; @@ -208,7 +216,7 @@ static void *thread_func(void *p) rcu_read_lock(); while (!atomic_read(&test_stop)) { - info->r = xorshift64star(info->r); + info->seed = xorshift64star(info->seed); info->func(info); } rcu_read_unlock(); @@ -221,7 +229,7 @@ static void *thread_func(void *p) static void prepare_thread_info(struct thread_info *info, int i) { /* seed for the RNG; each thread should have a different one */ - info->r = (i + 1) ^ time(NULL); + info->seed = (i + 1) ^ time(NULL); /* the first update will be a write */ info->write_op = true; /* the first resize will be down */ @@ -281,11 +289,25 @@ static void pr_params(void) static void do_threshold(double rate, uint64_t *threshold) { + /* + * For 0 <= rate <= 1, scale to fit in a uint64_t.

©2013 Xen Project, A Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation.
Xen Project is a trademark of The Linux Foundation.