Re: [Xen-devel] GFX Passthrough

[Pasi Kärkkäinen <pasik@xxxxxx>]

On Mon, May 10, 2010 at 04:36:18PM +0200, Tobias Geiger wrote:
> Hi List,
> 
> many People seem to be interested in the Graphic-Card Passthrough Feature 
> (for 
> more or less obvious reasons).
> 
> Official Support is still under development, and i hope not to interfere with 
> it 
> in any (bad) way ...
> 
> But i remember my own painfull and timeconsuming research when i wanted this 
> feature to work, so i thought perhaps this spares some time for the unpatient 
> users like me :)
> 
> So here is what i did to enable Passthrough of my Secondary PCIE Graphic-Card 
> which is a NVIDIA GT200;
> 
> Key thing seems to be the Patches 1 to 4
> (Patch #5 is only so that i have alsa-backed sound emulation , for some 
> reason 
> i have only oss-backed if not applied ?!)
> 
> Before all that, you need to read out the VGA Bios of the to-be-passthroughed-
> Graphiccard with a tool; i used nvflash.exe under DOS for this task.
> The patches assume you saved this file as "vgabios-pt.bin"
> 
> 1. check out xen unstable:
>     hg clone http://xenbits.xensource.com/xen-unstable.hg
> 2. go into there and do an "initial build" so that qemu-dm gets checked-out:
>     cd  xen-unstable.hg; cd tools; make ; make clean
> 3. apply the patches from within the "xen-unstable.hg" dir, with patch -p0
> 4. move the vgabios-pt.bin to "tools/firmware/vgabios/"
> 4. do a "make install"
> 
> Thats it. 
> 
> Warning: a "make clean" deletes the "vgabios-pt.bin" file , so remember to 
> put 
> it there again before your next "make" ...
> 
> After that, and assuming your hardware/bios is VT-D ready and your XEN-PCI 
> Config is correct (kernel-cmdline, xen-configfile), the HVM guest should be 
> able 
> to work with the passedthrough-Graphicscard.
> 
> Greetings, Good luck, and THANK YOU XEN-GUYS!
> Tobias
> 
> P.S.: NONE of the patches are my origin. they were gathered from here (xen-
> devel) and endless google-researches, representing the hard brainwork of much 
> smarter guys than me ;)


Thanks! I just added a link to this email to:
http://wiki.xensource.com/xenwiki/XenVGAPassthrough

-- Pasi

> --- tools/ioemu-remote/xen-setup.org  2010-03-15 12:07:28.650764455 +0100
> +++ tools/ioemu-remote/xen-setup      2010-03-15 12:06:35.217764207 +0100
> @@ -16,7 +16,7 @@ if test -z "${XEN_SCRIPT_DIR}"; then
>       XEN_SCRIPT_DIR="/etc/xen/scripts"
>  fi
>  
> -./configure --disable-gfx-check --disable-curses --disable-slirp "$@" 
> --prefix=${PREFIX}
> +./configure --audio-drv-list=alsa --enable-mixemu --disable-gfx-check 
> --disable-curses --disable-slirp "$@" --prefix=${PREFIX}
>  
>  target=i386-dm
>  

> --- tools/firmware/hvmloader/hvmloader.c.org  2010-03-15 11:59:29.517930657 
> +0100
> +++ tools/firmware/hvmloader/hvmloader.c      2010-03-15 12:04:59.486764339 
> +0100
> @@ -115,6 +115,9 @@ unsigned long pci_mem_end = PCI_MEM_END;
>  
>  static enum { VGA_none, VGA_std, VGA_cirrus, VGA_pt } virtual_vga = VGA_none;
>  
> +/* virtual BDF of pass-throughed gfx */
> +static uint8_t gfx_bdf;
> +
>  static void init_hypercalls(void)
>  {
>      uint32_t eax, ebx, ecx, edx;
> @@ -217,6 +220,42 @@ static void pci_setup(void)
>                  virtual_vga = VGA_cirrus;
>              else if ( virtual_vga == VGA_none )
>                  virtual_vga = VGA_pt;
> +                gfx_bdf = devfn;
> +
> +                /* Make vBAR=pBAR */
> +                printf("Make vBAR = pBAR of assigned gfx\n");
> +                for ( bar = 0; bar < 7; bar++ )
> +                {
> +                    bar_reg = PCI_BASE_ADDRESS_0 + 4*bar;
> +                    if ( bar == 6 )
> +                            bar_reg = PCI_ROM_ADDRESS;
> +                    /* When first time read, it will return physical address 
> */
> +                    bar_data = pci_readl(devfn, bar_reg);
> +                    pci_writel(devfn, bar_reg, bar_data);
> +
> +                    /* Now enable the memory or I/O mapping. */
> +                    cmd = pci_readw(devfn, PCI_COMMAND);
> +                    if ( (bar_reg == PCI_ROM_ADDRESS) ||
> +                             ((bar_data & PCI_BASE_ADDRESS_SPACE) ==
> +                              PCI_BASE_ADDRESS_SPACE_MEMORY) )
> +                          cmd |= PCI_COMMAND_MEMORY;
> +                    else
> +                          cmd |= PCI_COMMAND_IO;
> +                    cmd |= PCI_COMMAND_MASTER;
> +                    pci_writew(devfn, PCI_COMMAND, cmd);
> +               }
> +
> +                /* Map the interrupt. */
> +                pin = pci_readb(devfn, PCI_INTERRUPT_PIN);
> +                if ( pin != 0 )
> +                {
> +                    /* This is the barber's pole mapping used by Xen. */
> +                    link = ((pin - 1) + (devfn >> 3)) & 3;
> +                    isa_irq = pci_readb(PCI_ISA_DEVFN, 0x60 + link);
> +                    pci_writeb(devfn, PCI_INTERRUPT_LINE, isa_irq);
> +                }
> +                continue;
> +
>              break;
>          case 0x0680:
>              /* PIIX4 ACPI PM. Special device with special PCI config space. 
> */
> @@ -690,8 +729,10 @@ int main(void)
>          break;
>      case VGA_pt:
>          printf("Loading VGABIOS of passthroughed gfx ...\n");
> -        vgabios_sz =
> -            round_option_rom((*(uint8_t *)(VGABIOS_PHYSICAL_ADDRESS+2)) * 
> 512);
> +        memcpy((void *)VGABIOS_PHYSICAL_ADDRESS,
> +               vgabios_pt, sizeof(vgabios_pt));
> +        *(uint8_t *)(VGABIOS_PHYSICAL_ADDRESS + sizeof(vgabios_pt)) = 
> gfx_bdf;
> +        vgabios_sz = round_option_rom(sizeof(vgabios_pt) + 1);
>          break;
>      default:
>          printf("No emulated VGA adaptor ...\n");

> --- tools/firmware/hvmloader/Makefile.org     2010-03-15 11:59:04.617764082 
> +0100
> +++ tools/firmware/hvmloader/Makefile 2010-03-15 11:58:55.936761608 +0100
> @@ -50,6 +50,7 @@ hvmloader: $(OBJS) acpi/acpi.a
>  roms.h: ../rombios/BIOS-bochs-latest ../vgabios/VGABIOS-lgpl-latest.bin \
>       ../vgabios/VGABIOS-lgpl-latest.cirrus.bin ../etherboot/eb-roms.h
>       sh ./mkhex rombios ../rombios/BIOS-bochs-latest > roms.h
> +     sh ./mkhex vgabios_pt ../vgabios/vgabios-pt.bin >> roms.h         
>       sh ./mkhex vgabios_stdvga ../vgabios/VGABIOS-lgpl-latest.bin >> roms.h
>       sh ./mkhex vgabios_cirrusvga \
>               ../vgabios/VGABIOS-lgpl-latest.cirrus.bin >> roms.h

> --- tools/firmware/hvmloader/acpi/dsdt.asl.orig       2010-03-15 
> 06:50:35.791762654 +0100
> +++ tools/firmware/hvmloader/acpi/dsdt.asl    2010-03-15 12:03:37.579845691 
> +0100
> @@ -173,6 +173,34 @@ DefinitionBlock ("DSDT.aml", "DSDT", 2,
>                          0x00000000,
>                          0x00020000)
>  
> +                    /* reserve MMIO BARs of gfx for 1:1 mapping */
> +                    DWordMemory(
> +                        ResourceProducer, PosDecode, MinFixed, MaxFixed,
> +                        Cacheable, ReadWrite,
> +                        0x00000000,
> +                        0xE0000000,
> +                        0xEFFFFFFF,
> +                        0x00000000,
> +                        0x10000000)
> +
> +                    DWordMemory(
> +                        ResourceProducer, PosDecode, MinFixed, MaxFixed,
> +                        NonCacheable, ReadWrite,
> +                        0x00000000,
> +                        0xC0000000,
> +                        0xC1FFFFFF,
> +                        0x00000000,
> +                        0x02000000)
> +
> +                    DWordMemory(
> +                        ResourceProducer, PosDecode, MinFixed, MaxFixed,
> +                        NonCacheable, ReadWrite,
> +                        0x00000000,
> +                        0xC2000000,
> +                        0xC2FFFFFF,
> +                        0x00000000,
> +                        0x01000000)
> +
>                      DWordMemory(
>                          ResourceProducer, PosDecode, MinFixed, MaxFixed,
>                          Cacheable, ReadWrite,

> --- tools/ioemu-remote/hw/pass-through.c.org  2010-05-10 15:24:52.115083489 
> +0200
> +++ tools/ioemu-remote/hw/pass-through.c      2010-05-10 16:02:19.517997970 
> +0200
> @@ -1865,6 +1865,75 @@
>      return rc;
>  }
>  
> +#define PCI_HEADER_TYPE_ADDR        0x0e
> +#define PCI_BRIDGE_FLAG             0x01
> +
> +#define PCI_CLASS_CODE_ADDR_0       0x09
> +#define PCI_CLASS_CODE_ADDR_1       0x0a
> +#define PCI_CLASS_CODE_ADDR_2       0x0b
> +#define PCI_CLASS_CODE_DATA_0       0x060000
> +#define PCI_CLASS_CODE_DATA_1       0x0600
> +#define PCI_CLASS_CODE_DATA_2       0x06
> +
> +#define PCI_SECOND_BUS_NUMBER_ADDR  0x19
> +
> +#define PCI_BRIDGE_CONTROL_ADDR     0x3e
> +#define PCI_BRIDGE_VGA_ENABLE       0x18
> +
> +#define PCI_GRAPHIC_CONTROL_ADDR    0x52
> +#define PCI_HOST_BRIDGE_IGD_VGA_DISABLE 0x02
> +
> +/*
> + * Claim vga cycle for the graphics card pass-through
> + */
> +static uint32_t gfx_claim_vga_cycle(struct pci_access *pci_access,
> +       uint32_t bus, uint32_t devfn, uint32_t func)
> +{
> +    struct pci_dev *pci_dev;
> +
> +    for ( pci_dev = pci_access->devices; pci_dev != NULL; pci_dev = 
> pci_dev->next )
> +    {
> +        /* Check whether this is a ordinary bridge */
> +        if ( pci_read_byte(pci_dev, PCI_HEADER_TYPE_ADDR) == PCI_BRIDGE_FLAG 
> )
> +        {
> +            unsigned sec_bus_num = pci_read_byte(pci_dev, 
> PCI_SECOND_BUS_NUMBER_ADDR);
> +            unsigned ubrg = pci_read_byte(pci_dev, PCI_BRIDGE_CONTROL_ADDR);
> +
> +            PT_LOG("bridge for bus %d, previous bridge control is %x\n", 
> sec_bus_num, ubrg);
> +            PT_LOG("bus=0x%d, dev=0x%x, 
> func=0x%x\n",pci_dev->bus,pci_dev->dev,pci_dev->func);
> +
> +            if ( sec_bus_num == bus ) /* VGA device's bridge */
> +                ubrg |= PCI_BRIDGE_VGA_ENABLE;
> +            else /* Other device's bridge */
> +                ubrg &= ~PCI_BRIDGE_VGA_ENABLE;
> +
> +            pci_write_byte(pci_dev, PCI_BRIDGE_CONTROL_ADDR, ubrg);
> +            PT_LOG("bridge for bus %d, updated bridge control is %x\n", 
> sec_bus_num, ubrg);
> +        }
> +    }
> +
> +    for ( pci_dev = pci_access->devices; pci_dev != NULL; pci_dev = 
> pci_dev->next )
> +    {
> +        /* Check host bridge */
> +        if ( pci_read_word(pci_dev, PCI_CLASS_CODE_ADDR_1) == 
> PCI_CLASS_CODE_DATA_1 )
> +        {
> +            unsigned uigd = pci_read_byte(pci_dev, PCI_GRAPHIC_CONTROL_ADDR);
> +
> +            PT_LOG("previous igd control is %x\n", uigd);
> +
> +            if ( bus == 0 )
> +                uigd &= ~PCI_HOST_BRIDGE_IGD_VGA_DISABLE;
> +            else
> +                uigd |= PCI_HOST_BRIDGE_IGD_VGA_DISABLE;
> +
> +            pci_write_byte(pci_dev, PCI_GRAPHIC_CONTROL_ADDR, uigd);
> +            PT_LOG("updated igd control is %x\n", uigd);
> +        }
> +    }
> +
> +    return 0;
> +}
> +
>  /*
>   * register VGA resources for the domain with assigned gfx
>   */
> @@ -3298,6 +3367,8 @@
>  }
>  
>  /* read BAR */
> +static int gfx_first_read_BAR[7] = {1, 1, 1, 1, 1, 1, 1};
> +
>  static int pt_bar_reg_read(struct pt_dev *ptdev,
>          struct pt_reg_tbl *cfg_entry,
>          uint32_t *value, uint32_t valid_mask)
> @@ -3320,6 +3391,17 @@
>      /* use fixed-up value from kernel sysfs */
>      *value = ptdev->pci_dev->base_addr[index];
>  
> +    if ( ptdev->pci_dev->device_class == 0x300 )
> +    {
> +        if ( gfx_first_read_BAR[index] == 1 )
> +        {
> +            gfx_first_read_BAR[index] = 0;
> +            PT_LOG("first read BARs of gfx\n");
> +            return 0;
> +        }
> +    }
> +
> +
>      /* set emulate mask depend on BAR flag */
>      switch (ptdev->bases[index].bar_flag)
>      {
> @@ -4397,6 +4479,13 @@
>          }
>      }
>  
> +    if ( pci_dev->device_class == 0x0300 )
> +    {
> +        rc = gfx_claim_vga_cycle(pci_access, r_bus, r_dev, r_func);
> +        if ( rc != 0 )
> +            return NULL;
> +    }
> +
>      /* reinitialize each config register to be emulated */
>      rc = pt_config_init(assigned_device);
>      if ( rc < 0 ) {

> _______________________________________________
> Xen-devel mailing list
> Xen-devel@xxxxxxxxxxxxxxxxxxx
> http://lists.xensource.com/xen-devel


_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxxxxxxxx
http://lists.xensource.com/xen-devel