[Top] [All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

Re: [Xen-users] xen_blkbk times out before getting connected

I'm using the latest linux kernel 3.8.0 with Xen 4.3-unstable. Xenstored is being started at boot time. I did not see any message (dmesg/syslog/console) about device being plugged in but xenstore-ls shows me all 
the entries backend and frontend.

Here are some details
------------------------------
From /var/log/syslog:
Feb 3 10:37:31 localhost kernel: [ 36.466359] Event-channel device installed. Feb 3 10:37:31 localhost kernel: [ 36.636358] Register xen_blkbk_driver <<< custom debug message I added Feb 3 10:37:31 localhost kernel: [ 36.636367] xenbus_register_backend <<< custom debug message I added 
Feb  3 10:37:31 localhost kernel: [   36.683533] xenbus_register_backend
Feb 3 10:37:31 localhost kernel: [ 36.830115] xen-pciback: backend is vpci <<< custom debug message I added Feb 3 10:37:31 localhost kernel: [ 36.830123] xenbus_register_backend <<< custom debug message I added 
Feb  3 10:37:32 localhost xenstored: Checking store ...
Feb  3 10:37:32 localhost xenstored: Checking store complete.
Feb 3 10:37:32 localhost kernel: [ 37.075217] xenbus_probe: xenstored is ready <<< custom debug message I added Feb 3 10:37:32 localhost kernel: [ 37.075230] watching node FFFFFFFF81C7D1A0 
Feb  3 10:37:32 localhost kernel: [   37.081309] xenbus_scanf for node cpu/0
Feb 3 10:37:32 localhost kernel: [ 37.081327] XENBUS xen store gave: unknown error cpu 
Feb  3 10:37:32 localhost kernel: [   37.081333] xenbus_scanf for node cpu/1
Feb 3 10:37:32 localhost kernel: [ 37.085327] XENBUS unexpected type [17], expected [2] Feb 3 10:37:32 localhost kernel: [ 37.085342] watching node FFFFFFFF81C7D0C0 Feb 3 10:37:32 localhost kernel: [ 37.086857] XENBUS unexpected type [17], expected [4] Feb 3 10:37:32 localhost kernel: [ 37.086865] Failed to set shutdown watcher Feb 3 10:37:32 localhost kernel: [ 37.086880] backend_probe_and_watch - enumerates devices in xenstore and watches for changes <<< custom debug message I added Feb 3 10:37:32 localhost kernel: [ 37.089401] XENBUS unexpected type [4], expected [1] Feb 3 10:37:32 localhost kernel: [ 37.089411] watching node FFFFFFFF81C7D660 Feb 3 10:37:32 localhost kernel: [ 37.090715] XENBUS xen store gave: unknown error control/shutdown Feb 3 10:37:32 localhost kernel: [ 37.090725] frontend_probe_and_watch: calling xenbus_probe_devices <<< custom debug message I added Feb 3 10:37:32 localhost kernel: [ 37.092362] XENBUS unexpected type [17], expected [1] Feb 3 10:37:32 localhost kernel: [ 37.092376] watching node FFFFFFFF81C7D820 Feb 3 10:37:32 localhost kernel: [ 37.093334] watching node FFFFFFFF81C7DB20 Feb 3 10:37:32 localhost kernel: [ 37.094454] XENBUS xen store gave: unknown error backend <<< still not sure how critical this is Feb 3 10:37:32 localhost kernel: [ 37.094463] Failed to set balloon watcher <<< still not sure how critical this is 

> xenstore-ls
tool = ""
 xenstored = ""
local = ""
 domain = ""
  0 = ""
   data = ""
    meminfo_total = "930680"
    meminfo_free = "691868"
    os_name = "Ubuntu 12.04.1 LTS"
    os_majorver = "12"
    os_minorver = "04"
    os_uname = "3.8.0-rc5+"
    os_distro = "ubuntu"
    updated = "Sun Feb  3 10:43:32 MST 2013"
   name = "Domain-0"
   attr = ""
    xenbr0 = ""
     ip = "155.98.39.2"
    PVAddons = ""
     MajorVersion = "5"
     MinorVersion = "5"
     MicroVersion = "0"
     BuildVersion = "17913"
     Installed = "1"
   device-model = ""
    0 = ""
     state = "running"

After this I start the guest domain, here's the xenstore entries,

nodea:~> sudo xenstore-ls
tool = ""
 xenstored = ""
local = ""
 domain = ""
  0 = ""
   data = ""
    meminfo_total = "930680"
    meminfo_free = "695408"
    os_name = "Ubuntu 12.04.1 LTS"
    os_majorver = "12"
    os_minorver = "04"
    os_uname = "3.8.0-rc5+"
    os_distro = "ubuntu"
    updated = "Sun Feb  3 10:45:32 MST 2013"
   name = "Domain-0"
   attr = ""
    xenbr0 = ""
     ip = "155.98.39.2"
    PVAddons = ""
     MajorVersion = "5"
     MinorVersion = "5"
     MicroVersion = "0"
     BuildVersion = "17913"
     Installed = "1"
   device-model = ""
    0 = ""
     state = "running"
   memory = ""
    target = "1048576"
    static-max = "4294967292"
    freemem-slack = "314476"
   libxl = ""
    disable_udev = "1"
   backend = ""
    vbd = ""
     1 = ""
      2049 = ""
frontend = "/local/domain/1/device/vbd/2049" <<<< everything seems to be enumerated 
       params = "/dev/loop0"
       script = "/etc/xen/scripts/block"
       physical-device = "7:0"
       frontend-id = "1"
       online = "1"
       removable = "0"
       bootable = "1"
       state = "1"
       dev = "sda1"
       type = "phy"
       mode = "w"
       device-type = "disk"
  1 = ""
   vm = "/vm/9b0dfb10-f69d-4e22-9158-bb70456ece2e"
   name = "guest2"
   cpu = ""
    0 = ""
     availability = "online"
   memory = ""
    static-max = "512000"
    target = "512001"
    videoram = "-1"
   device = ""
    suspend = ""
     event-channel = ""
    vbd = ""
     2049 = ""
      backend = "/local/domain/0/backend/vbd/1/2049"
      backend-id = "0"
state = "1" <<<<< state changes to 5 and then after 5 secs it times out 
      virtual-device = "2049"
      device-type = "disk"
   control = ""
    shutdown = ""
    platform-feature-multiprocessor-suspend = "1"
    platform-feature-xs_reset_watches = "1"
   data = ""
   domid = "1"
   store = ""
    port = "1"
    ring-ref = "156845"
vm = ""
 9b0dfb10-f69d-4e22-9158-bb70456ece2e = ""
  uuid = "9b0dfb10-f69d-4e22-9158-bb70456ece2e"
  name = "guest2"
  image = ""
   ostype = "linux"
   kernel = "/boot/vmlinuz-3.8.0-rc5+"
   ramdisk = "/boot/initrd.img-3.8.0-rc5+"
   cmdline = "root=/dev/disk/by-path/xen-vbd-2049 ro xencons=tty"
  start_time = "1359913591.67"
libxl = ""
 1 = ""
  dm-version = "qemu_xen"

Thanks,
Yathi
On 2/3/2013 2:44 AM, Roger Pau Monné wrote:

On 03/02/13 03:07, Yathindra wrote:

Hi,

When ever I start a guest domain, backend driver times out since state
is not being connected. How do we debug such issues ?
I instrumented the backend drivers, specifically
xen/xenbus/xenbus_probe.c and xen/xenbus/xenbus_probe_backend.c to see
if the backend fails to probe the device. But I never saw any of the
messages I added in syslog or dmesg. I'm stuck on this problem
for weeks now. I'm not understanding why the backend fails to switch the
state.

Are you restarting the xenstore daemon? Also could you please post your
kernel version?

Did you get any messages from the kernel when you try to plug the device?


Heres the libxl log:

libxl: debug: libxl_device.c:313:libxl__device_disk_set_backend: Disk
vdev=sda1 spec.backend=unknown
libxl: debug: libxl_device.c:349:libxl__device_disk_set_backend: Disk
vdev=sda1, using backend phy
libxl: debug: libxl_create.c:821:initiate_domain_create: running bootloader
libxl: debug: libxl_bootloader.c:327:libxl__bootloader_run: no
bootloader configured, using user supplied kernel
libxl: debug: libxl_event.c:561:libxl__ev_xswatch_deregister: watch
w=0xc3a5e0: deregister unregistered
libxl: debug: libxl_numa.c:435:libxl__get_numa_candidate: New best NUMA
placement candidate found: nr_nodes=1, nr_cpus=2, nr_vcpus=3, free_memkb=997
libxl: detail: libxl_dom.c:192:numa_place_domain: NUMA placement
candidate with 1 nodes, 2 cpus and 997 KB free selected
domainbuilder: detail: xc_dom_allocate:
cmdline="root=/dev/disk/by-path/xen-vbd-2049 ro xencons=tty",
features="(null)"
libxl: debug: libxl_dom.c:393:libxl__build_pv: pv kernel mapped 0 path
/boot/vmlinuz-3.2.16
...
...
libxl: debug: libxl_event.c:512:libxl__ev_xswatch_register: watch
w=0xc3bd98 wpath=/local/domain/0/backend/vbd/1/2049/state token=3/0:
register slotnum=3
libxl: debug: libxl_create.c:1330:do_domain_create: ao 0xc39880:
inprogress: poller=0xc398e0, flags=i
libxl: debug: libxl_event.c:457:watchfd_callback: watch w=0xc3bd98
wpath=/local/domain/0/backend/vbd/1/2049/state token=3/0: event
epath=/local/domain/0/backend/vbd/1/2049/state
libxl: debug: libxl_event.c:600:devstate_watch_callback: backend
/local/domain/0/backend/vbd/1/2049/state wanted state 2 still waiting
state 1
libxl: debug: libxl_event.c:614:devstate_timeout: backend
/local/domain/0/backend/vbd/1/2049/state wanted state 2  timed out
libxl: debug: libxl_event.c:549:libxl__ev_xswatch_deregister: watch
w=0xc3bd98 wpath=/local/domain/0/backend/vbd/1/2049/state token=3/0:
deregister slotnum=3
libxl: debug: libxl_event.c:561:libxl__ev_xswatch_deregister: watch
w=0xc3bd98: deregister unregistered
libxl: error: libxl_device.c:943:device_backend_callback: unable to
connect device with path /local/domain/0/backend/vbd/1/2049
libxl: error: libxl_create.c:1077:domcreate_launch_dm: unable to add
disk devices
libxl: debug: libxl_event.c:512:libxl__ev_xswatch_register: watch
w=0xc3d868 wpath=/local/domain/0/backend/vbd/1/2049/state token=3/1:
register slotnum=3
libxl: debug: libxl_event.c:457:watchfd_callback: watch w=0xc3d868
wpath=/local/domain/0/backend/vbd/1/2049/state token=3/1: event
epath=/local/domain/0/backend/vbd/1/2049/state
libxl: debug: libxl_event.c:600:devstate_watch_callback: backend
/local/domain/0/backend/vbd/1/2049/state wanted state 6 still waiting
state 5
libxl: debug: libxl_event.c:614:devstate_timeout: backend
/local/domain/0/backend/vbd/1/2049/state wanted state 6  timed out
libxl: debug: libxl_event.c:549:libxl__ev_xswatch_deregister: watch
w=0xc3d868 wpath=/local/domain/0/backend/vbd/1/2049/state token=3/1:
deregister slotnum=3
libxl: debug: libxl_event.c:561:libxl__ev_xswatch_deregister: watch
w=0xc3d868: deregister unregistered
libxl: error: libxl_device.c:943:device_backend_callback: unable to
disconnect device with path /local/domain/0/backend/vbd/1/2049
libxl: error: libxl.c:1725:devices_destroy_cb: libxl__devices_destroy
failed for 1
libxl: debug: libxl_event.c:1497:libxl__ao_complete: ao 0xc39880:
complete, rc=-3
libxl: debug: libxl_event.c:1469:libxl__ao__destroy: ao 0xc39880: destroy
xc: debug: hypercall buffer: total allocations:366 total releases:366
xc: debug: hypercall buffer: current allocations:0 maximum allocations:4
xc: debug: hypercall buffer: cache current size:4
xc: debug: hypercall buffer: cache hits:356 misses:4 toobig:6

Thanks,
Yathi

_______________________________________________
Xen-users mailing list
Xen-users@xxxxxxxxxxxxx
http://lists.xen.org/xen-users




_______________________________________________
Xen-users mailing list
Xen-users@xxxxxxxxxxxxx
http://lists.xen.org/xen-users

 

©2013 Xen Project, A Linux Foundation Collaborative Project. All Rights Reserved.
Linux Foundation is a registered trademark of The Linux Foundation.
Xen Project is a trademark of The Linux Foundation.

Rackspace

Lists.xenproject.org is hosted with RackSpace, monitoring our
servers 24x7x365 and backed by RackSpace's Fanatical Support®.