VM in NAT network setup can't ping to internet

[Yang <teddyyyy123@xxxxxxxxx>]

my host has IP 192.168.0.105, it goes to internet through my home wireless router 192.168.0.1

on the host, to setup NAT for my VMs , I created a new bridge by the steps in http://blog.manula.org/2012/04/manually-configuring-nat-networking-in.html

note that in the following there is a xenbr0, that is used for the other VM in a bridged network setting, it's not related to this NAT usage. I tried not setting up the xenbr0, the issues with my NAT remains.

root@yy-70A4000HUX:/home/yy#      brctl addbr br
root@yy-70A4000HUX:/home/yy#      ifconfig br 10.0.0.1 up
root@yy-70A4000HUX:/home/yy# ifconfig
anbox0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.250.1  netmask 255.255.255.0  broadcast 0.0.0.0
        inet6 fe80::f47c:9dff:fe07:863f  prefixlen 64  scopeid 0x20<link>
        ether f6:7c:9d:07:86:3f  txqueuelen 1000  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 268  bytes 96507 (96.5 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

br: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.1  netmask 255.0.0.0  broadcast 10.255.255.255
        inet6 fe80::4:3ff:fee2:f029  prefixlen 64  scopeid 0x20<link>
        ether 02:04:03:e2:f0:29  txqueuelen 1000  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 35  bytes 13838 (13.8 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

br-1870818b7486: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 172.21.0.1  netmask 255.255.0.0  broadcast 172.21.255.255
        inet6 fe80::42:53ff:fe65:ffa2  prefixlen 64  scopeid 0x20<link>
        ether 02:42:53:65:ff:a2  txqueuelen 0  (Ethernet)
        RX packets 11381  bytes 5398144 (5.3 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 29185  bytes 41434853 (41.4 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

br-c877ad85e968: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 172.24.0.1  netmask 255.255.0.0  broadcast 172.24.255.255
        ether 02:42:dc:25:ae:71  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

docker0: flags=4099<UP,BROADCAST,MULTICAST>  mtu 1500
        inet 172.17.0.1  netmask 255.255.0.0  broadcast 172.17.255.255
        ether 02:42:28:3a:59:0f  txqueuelen 0  (Ethernet)
        RX packets 0  bytes 0 (0.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 0  bytes 0 (0.0 B)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        ether 44:39:c4:54:25:d3  txqueuelen 1000  (Ethernet)
        RX packets 49625  bytes 47041255 (47.0 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 24247  bytes 6906668 (6.9 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
        device interrupt 20  memory 0xf7c00000-f7c20000  

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 183934  bytes 257676802 (257.6 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 183934  bytes 257676802 (257.6 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

veth5a03a3c: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::a01c:a4ff:feb4:30d8  prefixlen 64  scopeid 0x20<link>
        ether a2:1c:a4:b4:30:d8  txqueuelen 0  (Ethernet)
        RX packets 2285  bytes 157539 (157.5 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 4333  bytes 616718 (616.7 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vethea8d130: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::5802:52ff:fee4:839e  prefixlen 64  scopeid 0x20<link>
        ether 5a:02:52:e4:83:9e  txqueuelen 0  (Ethernet)
        RX packets 47758  bytes 10697729 (10.6 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 61672  bytes 50466361 (50.4 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vethfa086dd: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::f8b9:9fff:fe86:7f33  prefixlen 64  scopeid 0x20<link>
        ether fa:b9:9f:86:7f:33  txqueuelen 0  (Ethernet)
        RX packets 30149  bytes 8853899 (8.8 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 32553  bytes 4712623 (4.7 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vif2.0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        ether fe:ff:ff:ff:ff:ff  txqueuelen 32  (Ethernet)
        RX packets 1833  bytes 89740 (89.7 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 7823  bytes 1554944 (1.5 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

xenbr0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.0.105  netmask 255.255.255.0  broadcast 192.168.0.255
        inet6 fe80::4639:c4ff:fe54:25d3  prefixlen 64  scopeid 0x20<link>
        ether 44:39:c4:54:25:d3  txqueuelen 1000  (Ethernet)
        RX packets 43873  bytes 44476176 (44.4 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 19454  bytes 6448249 (6.4 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0


veth5a03a3c: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::a01c:a4ff:feb4:30d8  prefixlen 64  scopeid 0x20<link>
        ether a2:1c:a4:b4:30:d8  txqueuelen 0  (Ethernet)
        RX packets 2260  bytes 155817 (155.8 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 4278  bytes 609680 (609.6 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vethea8d130: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::5802:52ff:fee4:839e  prefixlen 64  scopeid 0x20<link>
        ether 5a:02:52:e4:83:9e  txqueuelen 0  (Ethernet)
        RX packets 47269  bytes 10634147 (10.6 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 61229  bytes 50356270 (50.3 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vethfa086dd: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet6 fe80::f8b9:9fff:fe86:7f33  prefixlen 64  scopeid 0x20<link>
        ether fa:b9:9f:86:7f:33  txqueuelen 0  (Ethernet)
        RX packets 29737  bytes 8749262 (8.7 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 32107  bytes 4648615 (4.6 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

vif2.0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        ether fe:ff:ff:ff:ff:ff  txqueuelen 32  (Ethernet)
        RX packets 1803  bytes 88288 (88.2 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 7724  bytes 1544338 (1.5 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

xenbr0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 192.168.0.105  netmask 255.255.255.0  broadcast 192.168.0.255
        inet6 fe80::4639:c4ff:fe54:25d3  prefixlen 64  scopeid 0x20<link>
        ether 44:39:c4:54:25:d3  txqueuelen 1000  (Ethernet)
        RX packets 43602  bytes 44431466 (44.4 MB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 19325  bytes 6431404 (6.4 MB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

the following is iptables setup:

root@yy-70A4000HUX:/home/yy#       iptables -A FORWARD --in-interface br -j ACCEPT
root@yy-70A4000HUX:/home/yy#      iptables --table nat -A POSTROUTING --out-interface eth0 -j MASQUERADE
root@yy-70A4000HUX:/home/yy# iptables -t nat -L
Chain PREROUTING (policy ACCEPT)
target     prot opt source               destination        
DOCKER     all  --  anywhere             anywhere             ADDRTYPE match dst-type LOCAL

Chain INPUT (policy ACCEPT)
target     prot opt source               destination        

Chain OUTPUT (policy ACCEPT)
target     prot opt source               destination        
DOCKER     all  --  anywhere            !localhost/8          ADDRTYPE match dst-type LOCAL

Chain POSTROUTING (policy ACCEPT)
target     prot opt source               destination        
MASQUERADE  all  --  172.17.0.0/16        anywhere            
MASQUERADE  all  --  172.24.0.0/16        anywhere            
MASQUERADE  all  --  172.21.0.0/16        anywhere            
MASQUERADE  all  --  192.168.250.0/24    !192.168.250.0/24     /* managed by anbox-bridge */
MASQUERADE  tcp  --  172.21.0.4           172.21.0.4           tcp dpt:http-alt
MASQUERADE  all  --  anywhere             anywhere            

Chain DOCKER (2 references)
target     prot opt source               destination        
RETURN     all  --  anywhere             anywhere            
RETURN     all  --  anywhere             anywhere            
RETURN     all  --  anywhere             anywhere            
DNAT       tcp  --  anywhere             anywhere             tcp dpt:http-alt to:172.21.0.4:8080
root@yy-70A4000HUX:/home/yy#

the VM is created from this config:

root@yy-70A4000HUX:/home/yy/ub_xen/1# cat run.cfg
name = "ub1"

#kernel = "/home/yy/ub_xen/1/vmlinuz"
#ramdisk = "/home/yy/ub_xen/1/initrd.gz"
bootloader = "pygrub"

memory = 1024
vcpus = 1

# Custom option for Open vSwitch
vif=['bridge=br,mac=00:16:3e:66:8c:25']

#vfb = [ 'type=vnc' ]

disk = ['tap:aio:/home/yy/ub_xen/1/vmdisk0,xvda,w' ]

# You may also consider some other options
# [[http://xenbits.xen.org/docs/4.4-testing/man/xl.cfg.5.html]]

 

now I setup the "eth0 " inside VM

yy@ubuntu:~$ sudo      route add default gw 10.0.0.1
yy@ubuntu:~$
yy@ubuntu:~$ ifconfig
eth0: flags=4163<UP,BROADCAST,RUNNING,MULTICAST>  mtu 1500
        inet 10.0.0.2  netmask 255.0.0.0  broadcast 10.255.255.255
        inet6 fe80::216:3eff:fe66:8c25  prefixlen 64  scopeid 0x20<link>
        ether 00:16:3e:66:8c:25  txqueuelen 1000  (Ethernet)
        RX packets 7  bytes 702 (702.0 B)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 41  bytes 8166 (8.1 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

lo: flags=73<UP,LOOPBACK,RUNNING>  mtu 65536
        inet 127.0.0.1  netmask 255.0.0.0
        inet6 ::1  prefixlen 128  scopeid 0x10<host>
        loop  txqueuelen 1000  (Local Loopback)
        RX packets 1386  bytes 99578 (99.5 KB)
        RX errors 0  dropped 0  overruns 0  frame 0
        TX packets 1386  bytes 99578 (99.5 KB)
        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0

by this time, the VM can ping the host (either 10.0.0.1 or 192.168.0.105), but can't ping 192.168.0.1 (which needs to go outside the host)