Re: [Xen-devel] [OSSTEST PATCH 2/4] Build XEN and HVM Dom0 kernel for L1 guest VM

["Hu, Robert" <robert.hu@xxxxxxxxx>]

> -----Original Message-----
> From: Wei Liu [mailto:wei.liu2@xxxxxxxxxx]
> Sent: Thursday, December 11, 2014 7:25 PM
> To: Pang, LongtaoX
> Cc: xen-devel@xxxxxxxxxxxxx; Ian.Jackson@xxxxxxxxxxxxx;
> Ian.Campbell@xxxxxxxxxx; wei.liu2@xxxxxxxxxx; Hu, Robert; Zheng, Di
> Subject: Re: [OSSTEST PATCH 2/4] Build XEN and HVM Dom0 kernel for L1 guest
> VM
> 
> On Wed, Dec 10, 2014 at 04:07:38PM +0800, longtao.pang wrote:
> > From: "longtao.pang" <longtaox.pang@xxxxxxxxx>
> >
> > This patch is used for building XEN and HVM Dom0 kernel for L1 guest VM,
> > and then reboot L1 guest into xen kernel.
> >
> 
> I think you can just use the L0 Xen and Dom0 kernel, that would save you
> lots of time running this test case. It can also help simplifies this
> patch, maybe?
> 
> > ---
> >  sg-run-job     |    1 +
> >  ts-xen-install |  149
> +++++++++++++++++++++++++++++++++++++++++---------------
> >  2 files changed, 111 insertions(+), 39 deletions(-)
> >
> > diff --git a/sg-run-job b/sg-run-job
> > index 8dcf7af..e513bd1 100755
> > --- a/sg-run-job
> > +++ b/sg-run-job
> > @@ -291,6 +291,7 @@ proc run-job/test-pair {} {
> >  proc need-hosts/test-nested {} {return host}
> >  proc run-job/test-nested {} {
> >      run-ts . = ts-debian-hvm-install + host + nested + nested_L1
> > +    run-ts . = ts-xen-install + host + nested + nested_build
> >  }
> >
> >  proc test-guest-migr {g} {
> > diff --git a/ts-xen-install b/ts-xen-install
> > index 4d34d1f..c175d6d 100755
> > --- a/ts-xen-install
> > +++ b/ts-xen-install
> > @@ -28,19 +28,25 @@ use Osstest::CXFabric;
> >  my $checkmode= 0;
> >
> >  tsreadconfig();
> > -
> > +our $w_ho;
> >  our @hos;
> > -
> > -if (@ARGV and $ARGV[0] eq '--check') {
> > -    $checkmode= 1;
> > -    shift @ARGV;
> > -    logm("checking builds are done...");
> > +our ($whhost,$gn,$nested_build) = @ARGV;
> > +$nested_build ||= '';
> > +if ($nested_build eq 'nested_build') {
> > +    $whhost ||= 'host';
> > +    $gn ||= 'nested';
> >  } else {
> > -    if (!@ARGV) {
> > -   push @ARGV, 'host';
> > -    }
> > -    foreach my $k (@ARGV) {
> > -        push @hos, selecthost($k);
> > +    if (@ARGV and $ARGV[0] eq '--check') {
> > +        $checkmode= 1;
> > +        shift @ARGV;
> > +        logm("checking builds are done...");
> > +    } else {
> > +        if (!@ARGV) {
> > +            push @ARGV, 'host';
> > +        }
> > +        foreach my $k (@ARGV) {
> > +            push @hos, selecthost($k);
> > +        }
> >      }
> >  }
> >
> > @@ -49,18 +55,18 @@ our $ho;
> >  my %distpath;
> >
> >  sub packages () {
> > -    target_install_packages($ho,
> > +    target_install_packages($w_ho,
> >                              qw(bridge-utils vncsnapshot libaio1
> libpixman-1-0
> >                                 libsdl1.2debian libglib2.0-0
> liblzma5));
> > -    target_install_packages($ho,
> > +    target_install_packages($w_ho,
> >                         $ho->{Suite} =~ /squeeze/ ? "libyajl1" : 
> > "libyajl2");
> >      if ($ho->{Suite} !~ m/lenny|squeeze/) {
> > -        target_install_packages($ho, 'libfdt1');
> > +        target_install_packages($w_ho, 'libfdt1');
> >      }
> >      if ($r{arch} eq 'i386') {
> > -   target_install_packages($ho, 'libc6-xen');
> > +   target_install_packages($w_ho, 'libc6-xen');
> >      }
> > -    target_install_packages($ho, @{toolstack()->{ExtraPackages}})
> > +    target_install_packages($w_ho, @{toolstack()->{ExtraPackages}})
> >          if toolstack()->{ExtraPackages};
> >  }
> >
> > @@ -69,14 +75,14 @@ sub extract () {
> >      push @parts, 'libvirt' if $r{toolstack} eq "libvirt";
> >
> >      foreach my $part (@parts) {
> > -        target_extract_jobdistpath($ho, $part, "path_${part}dist",
> > +        target_extract_jobdistpath($w_ho, $part, "path_${part}dist",
> >                                $r{"${part}buildjob"}, \%distpath);
> >      }
> > -    target_cmd_root($ho, '/sbin/ldconfig');
> > +    target_cmd_root($w_ho, '/sbin/ldconfig');
> >  }
> >
> >  sub adjustconfig () {
> > -    target_editfile_root($ho, "/etc/xen/xend-config.sxp",
> > +    target_editfile_root($w_ho, "/etc/xen/xend-config.sxp",
> >                      "xend-config.sxp", sub {
> >     my (@domains) = (qw(localhost localhost.localdomain),
> >                      ".".$c{DnsDomain}, ".".$c{TestHostDomain});
> > @@ -108,13 +114,13 @@ sub adjustconfig () {
> >                          /etc/sysconfig/xencommons
> >                          /etc/default/xend
> >                          /etc/sysconfig/xend)) {
> > -        next unless target_file_exists($ho, $try);
> > +        next unless target_file_exists($w_ho, $try);
> >          $trace_config_file= $try;
> >          last;
> >      }
> >      die unless defined $trace_config_file;
> >
> > -    target_editfile_root($ho, $trace_config_file, sub {
> > +    target_editfile_root($w_ho, $trace_config_file, sub {
> >          my $prnow;
> >          $prnow= sub {
> >              print EO "XENCONSOLED_TRACE=guest\n" or die $!;
> > @@ -128,7 +134,7 @@ sub adjustconfig () {
> >          $prnow->();
> >      });
> >
> > -    target_cmd_root($ho, 'mkdir -p /var/log/xen/console');
> > +    target_cmd_root($w_ho, 'mkdir -p /var/log/xen/console');
> >
> >      setup_cxfabric($ho);
> >  }
> > @@ -156,19 +162,19 @@ sub setupboot () {
> >      $xenhopt .= " $append" if defined $append;
> >
> >      my @hooks;
> > -
> > +
> >      if (host_involves_pcipassthrough($ho)) {
> >          push @hooks, {
> >              EditBootOptions => sub {
> >                  my ($ho,$hopt,$kopt) = @_;
> >                  $$hopt .= ' iommu=on';
> >                  my $hide= ' xen-pciback.hide='. join '',map
> { "($_->{Bdf})" }
> > -                    host_get_pcipassthrough_devs($ho);
> > +                host_get_pcipassthrough_devs($ho);
> >                  logm("pci passthrough: hiding in dom0: $hide");
> >                  $$kopt .= $hide;
> > -            }
> > -        };
> > -    }
> > +                }
> > +            };
> > +        }
> >
> >      my $want_kernver = get_runvar('kernel_ver',$r{'kernbuildjob'});
> >      debian_boot_setup($ho, $want_kernver, $xenhopt, \%distpath,
> \@hooks);
> > @@ -182,7 +188,7 @@ sub setupinitd () {
> >      my $ts= toolstack();
> >      my $xencommons= '/etc/init.d/xencommons';
> >      my $have_xencommons=
> > -        !!target_cmd_output_root($ho, <<END);
> > +        !!target_cmd_output_root($w_ho, <<END);
> >   if test -f $xencommons && ! grep 'FOR USE WITH LIBXL'
> $xencommons >/dev/null
> >   then
> >     echo y
> > @@ -211,7 +217,33 @@ END
> >          $updatercd->($initd,93) if defined $initd;
> >          $updatercd->('xenbridge',38) if $ts->{OldSeparateBridgeInitd};
> >      }
> > -    target_cmd_root($ho, $cmd);
> > +    target_cmd_root($w_ho, $cmd);
> > +}
> > +
> > +sub setup_l1_bridge($)
> > +{
> > +    my ($ho)=@_;
> > +    my $bridge_port;
> > +    my $route_output=target_cmd_output_root($ho,"route -n");
> > +    foreach my $line (split /\n/, $route_output){
> > +        if($line =~ m/^\s*(?:(?:0\.0\.0\.0)|default).*\s(\w+)\s*$/ai){
> > +            $bridge_port=$1;
> > +            logm("get L1 bridge phy port $bridge_port");
> > +            last;
> > +        }
> > +    }
> > +        die "cannot find L1 port for xenbr0 bridge" if !defined
> $bridge_port;
> > +
> > +    target_editfile_root($ho, "/etc/network/interfaces",
> > +                         "etc-network-interfaces",
> > +                        sub {
> > +                            while(<EI>){
> > +
> s/^\s*iface\s*$bridge_port\s*inet.*dhcp\s*$/iface $bridge_port inet
> manual\nauto xenbr0\niface xenbr0 inet dhcp\n\tbridge_ports
> $bridge_port\n/;
> > +
> s/^\s*auto\s*$bridge_port/#auto\t$bridge_port/;
> > +                                print EO;
> > +                            }
> > +                         });
> > +    target_cmd_root($ho,"brctl addbr xenbr0; brctl addif xenbr0
> $bridge_port; init 6");
> >  }
> 
> FWIW, OSSTest has a bunch of overlay files (look at overlay directory),
> which includes an init script called xenbridge. In theory if you're
> reusing this script (ts-xen-install) then you don't need to worry about
> setting up bridge?
I tried this approach, using xenbridge init scripts, it can work.
However, in original xen install, it seems not used. I don't see 'xenbridge'
in /etc/rc2.d/
root@osstest-host2:/etc/rc2.d# ls /etc/rc2.d/
README   S13rpcbind     S16rsyslog  S18acpid  S18cron  S18exim4  S18rsync  
S18xencommons  S21osstest-confirm-booted  S21rmnologin
S01motd  S14nfs-common  S17apache2  S18atd    S18dbus  S18ntp    S18ssh    
S20bootlogs    S21rc.local

Here is the related piece of code, from ts-xen-install

sub setupinitd () {
    my $ts= toolstack();
    my $xencommons= '/etc/init.d/xencommons';
    my $have_xencommons=
        !!target_cmd_output_root($w_ho, <<END);
 if test -f $xencommons && ! grep 'FOR USE WITH LIBXL' $xencommons >/dev/null
 then
   echo y
 fi
END
    $initscripts_nobridge= !defined($ts->{OldDaemonInitd}) || $have_xencommons;
    logm("init.d scripts ".
         ($initscripts_nobridge
          ? 'do not mess with bridge, doing it in interfaces(5)'
          : '_do_ mess with bridge, letting them handle it'));
    my $cmd= '';
    my $updatercd= sub {
        my ($script,$start) = @_;
        $cmd .= "\n    update-rc.d $script start $start 2 .";
    };
    if ($initscripts_nobridge) {
        my $script= $have_xencommons ? 'xencommons' : 'xenlightdaemons';
        $updatercd->($script,92);
        my $pri= 93;
        foreach my $d (@{ $ts->{NewDaemons} }) {
            $updatercd->("$d",$pri);
            $pri++;
        }
    } else {
        my $initd= $ts->{OldDaemonInitd};
        $updatercd->($initd,93) if defined $initd;
        $updatercd->('xenbridge',38) if $ts->{OldSeparateBridgeInitd};
    }
    target_cmd_root($w_ho, $cmd);
}

seems as long as xencommons is there, the $initscripts_nobridge will be true, 
i.e
xenbridge service will not be inited.

But normal ts-xen-install indeed created a bridge named 'xenbr0' (xenbridge 
will named
it to 'peth0' if called).
Shall I know the xenbr0 is created? I think not by xenbridge init service.
> 
> >
> >  sub nodhcp () {
> > @@ -322,17 +354,56 @@ sub forbidden () {
> >  END
> >  }
> >
> > -if ($checkmode) {
> > -    extract();
> > -} else {
> > -    die if @hos > 1;
> > -    $ho= $hos[0];
> > -
> > +if ($nested_build eq 'nested_build') {
> > +    our $gho;
> > +    $ho= selecthost($whhost);
> > +    $gho= selectguest($gn,$ho);
> > +    $w_ho = $gho;
> > +    store_runvar("$gho->{Guest}_kernkind",$r{'kernkind'});
> > +    $gho->{Suite}=$ho->{Suite};
> > +
> > +    guest_check_ip($gho);
> >      packages();
> >      extract();
> > -    forbidden();
> >      adjustconfig();
> > -    setupboot();
> > -    setupinitd();
> > -    nodhcp();
> > +    my $want_kernver = get_runvar('kernel_ver',$r{'kernbuildjob'});
> > +    my $bootloader;
> > +    $bootloader=setupboot_grub2($gho, $want_kernver, "");
> > +
> > +
> > +    target_cmd_root($gho,
> > +                    "update-initramfs -k $want_kernver -c ||".
> > +                    " update-initramfs -k $want_kernver -u",
> > +                    200);
> > +    $bootloader->{UpdateConfig}($gho); #so that /boot/grub/grub.cfg have
> new kernel and xen
> > +    $bootloader->{PreFinalUpdate}();        #update /etc/default/grub,
> by setting default entry we want
> > +
> > +    setupinitd ();
> > +    $bootloader->{UpdateConfig}($gho);      #use the default entry,
> apply it to /boot/grub/grub.cfg
> > +    guest_editconfig($gho->{Host}, $gho, sub {
> > +        s/#nestedhvm/nestedhvm/;
> > +        });
> > +    target_cmd_root($gho,"sync");
> > +    setup_l1_bridge($gho);          #after setup L1 bridge, it will
> reboot for network settiings to take effect
> > +    logm("ready to reboot L1 Xen");
> > +    guest_await($gho, target_var($gho,'boot_timeout'));
> > +    guest_check_up($gho);
> > +
> 
> This hunk is copied from Debian.pm. If debian_setup_boot cannot meet
> your requirement, please refactor it.
> 
> Wei.
> 
> > +
> > +} else {
> > +
> > +    if ($checkmode) {
> > +        extract();
> > +    } else {
> > +        die if @hos > 1;
> > +        $ho= $hos[0];
> > +        $w_ho = $ho;
> > +        packages();
> > +        extract();
> > +        forbidden();
> > +        adjustconfig();
> > +        setupboot();
> > +        setupinitd();
> > +        nodhcp();
> > +    }
> >  }
> > --
> > 1.7.10.4

_______________________________________________
Xen-devel mailing list
Xen-devel@xxxxxxxxxxxxx
http://lists.xen.org/xen-devel