Install OpenNebula-2.2-1 with Xen
Reference
http://opennebula.org
http://downloads.dsa-research.org/opennebula
Environments
Tested environments info
Test System / Network
- On VMware ESXi
- One vSwitch
- Two VMs
- VM IP Range : 192.168.100.151~160 (10EA)
- By root privileges
- Run "yum -y update" on all nodes
- Template VM Image : CentOS 5.6 x86_64
Test Nodes
- Management Node
- IP : 192.168.100.121 (one-01)
- OS : CentOS 5.6 x86_64 (2.6.18-238.9.1.el5)
- 1Core / 384RAM
- Xen Node IP : 192.168.100.122
- IP : 192.168.100.122 (one-02)
- OS : CentOS 5.6 x86_64 (2.6.18-238.19.1.el5xen)
- 2Core / 1024RAM
On whole nodes
Add to /etc/hosts
192.168.100.121 one-01
192.168.100.122 one-02
On Management Node
Install OpenNebula & Setup Env.
# cd /usr/local/src
Download opennebula-2.2-1.x86_64.rpm to /usr/local/src/ (from http://downloads.dsa-research.org/opennebula)
# wget ftp://ftp.pbone.net/mirror/centos.karan.org/el5/extras/testing/SRPMS/xmlrpc-c-1.06.18-1.el5.kb.src.rpm
# ls -al /usr/local/src/
-rw-r--r-- 1 root root 717867 Jul 22 13:53 opennebula-2.2-1.x86_64.rpm
-rw-r--r-- 1 root root 708799 Jul 23 21:49 xmlrpc-c-1.06.18-1.el5.kb.src.rpm
# yum install ruby ruby-devel ruby-docs ruby-ri ruby-irb ruby-rdoc
# rpmbuild --rebuild xmlrpc-c-1.06.18-1.el5.kb.src.rpm
# rpm -Uvh /usr/src/redhat/RPMS/x86_64/xmlrpc-c-*
# rpm -Uvh opennebula-2.2-1.x86_64.rpm
Setup OpenNebula Admin User & Env
# echo "export ONE_XMLRPC=http://localhost:2633/RPC2" >> /etc/profile
# echo "export ONE_AUTH=/home/oneadmin/one_auth" >> /etc/profile
# useradd oneadmin -d /home/oneadmin
# echo "oneadmin:1234" > /home/oneadmin/one_auth
SSH Key Create & Setup sudo
# mkdir /home/oneadmin/.ssh
# ssh-keygen -t rsa -N '' -f /home/oneadmin/.ssh/id_rsa
# cp -a /home/oneadmin/.ssh/id_rsa.pub /home/oneadmin/.ssh/authorized_keys
# chmod 700 /home/oneadmin/.ssh
# chown -R oneadmin:oneadmin /home/oneadmin/.ssh
# sed -i 's/^Defaults requiretty/# /g' /etc/sudoers
Add to /etc/ssh/ssh_config
Host *
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
On Xen Node
Install Xen
# yum install ruby ebtables
# yum groupinstall Vritualization
# useradd oneadmin -d /home/oneadmin
# reboot
# virsh net-destroy default
# virsh net-undefine default
SSH Key Create & Setup sudo
# mkdir /home/oneadmin/.ssh
# scp root@one-01:/home/oneadmin/.ssh/* /home/oneadmin/.ssh/
# chonw -R oneadmin:oneadmin /home/oneadmin/.ssh
# sed -i 's/^Defaults requiretty/# /g' /etc/sudoers
Add to /etc/ssh/ssh_config
Host *
StrictHostKeyChecking no
UserKnownHostsFile /dev/null
on Xen Node
Create Xen Template Image
# virt-install --name centos5_64 --ram 384 --paravirt --location=http://ftp.daum.net/centos/5/os/x86_64/ --file=/home/oneadmin/centos5_64.xen.img -s 1.5
** Install Minimal Package
On VM Template Image - Setup VM Env. (After Installation & Reboot)
# echo "/var/lib/one/one-boot-init.sh" >> /etc/rc.local
# mkdir /var/lib/one/
# vi /var/lib/one/one-boot-init.sh
#!/bin/bash
DEV_CD="/dev/xvdc"
MNT="media/one"
CHK_FILE="/var/lib/one/.done_context"
if [ -b ${DEV_CD} ]; then
mount -t iso9660 ${DEV_CD} ${MNT}
if [ -f ${MNT}/context.sh ]; then
. ${MNT}/init.sh
fi
umount ${MNT}
if [ ! -e ${CHK_FILE} ]; then
touch ${CHK_FILE}
chattr +i ${CHK_FILE}
fi
fi
exit 0
# vi /etc/sysconfig/network-scripts/ifcfg-eth0
# Xen Virtual Ethernet
DEVICE=eth0
BOOTPROTO=static
ONBOOT=no
# vi /etc/sysconfig/network
NETWORKING=yes
NETWORKING_IPV6=no
HOSTNAME=localhost.localdomain
- Default Runlevel is 3 (/etc/inittab)
- Auto Started Service is Compact (chkconfig & ntsysv)
# sed -i 's/^id:.*/id:3:initdefault:/g' /etc/inittab
# chkconfig --list | grep "3:on"
acpid 0:off 1:off 2:on 3:on 4:on 5:on 6:off
cpuspeed 0:off 1:on 2:on 3:on 4:on 5:on 6:off
crond 0:off 1:off 2:on 3:on 4:on 5:on 6:off
haldaemon 0:off 1:off 2:off 3:on 4:on 5:on 6:off
iptables 0:off 1:off 2:on 3:on 4:on 5:on 6:off
irqbalance 0:off 1:off 2:on 3:on 4:on 5:on 6:off
messagebus 0:off 1:off 2:off 3:on 4:on 5:on 6:off
network 0:off 1:off 2:on 3:on 4:on 5:on 6:off
sshd 0:off 1:off 2:on 3:on 4:on 5:on 6:off
syslog 0:off 1:off 2:on 3:on 4:on 5:on 6:off
- Copy Template Image to Management Node
# scp /home/oneadmin/centos5_64.xen.img root@one-01:/home/oneadmin/
On Management Node
Setup : NFS Export (for VM_DIR)
# vi /etc/exports
/var/lib/one 192.168.100.122(rw,no_root_squash)
/usr/share/one/hooks 192.168.100.122(rw,no_root_squash)
# exportfs -avr
# chkconfig portmap on
# chkconfig nfs on
# chkconfig nfslock on
# service portmap start
# service nfs start
# service nfslock start
On Xen Node
Mount NFS
# echo "one-01:/var/lib/one/ /var/lib/one nfs defaults 0 0" >> /etc/fstab
# echo "one-01:/usr/share/one/hooks /usr/share/one/hooks nfs defaults 0 0" >> /etc/fstab
# mkdir -p /usr/share/one/hooks
# mount -a
On Management Node
Configuration OpenNebula (oned.conf)
# vi /etc/one/oned.conf
HOST_MONITORING_INTERVAL = 60
VM_POLLING_INTERVAL = 60
SCRIPTS_REMOTE_DIR=/var/tmp/one
PORT=2633
DB = [ backend = "mysql", server = "localhost", port = "3306", user = "root", passwd = "1234", db_name = "one" ]
VNC_BASE_PORT = 5900
DEBUG_LEVEL=3
NETWORK_SIZE = 254
MAC_PREFIX = "02:00"
DEFAULT_IMAGE_TYPE = "OS"
DEFAULT_DEVICE_PREFIX = "xvd"
IM_MAD = [
name = "im_xen",
executable = "one_im_ssh",
arguments = "xen" ]
VM_MAD = [
name = "vmm_xen",
executable = "one_vmm_ssh",
arguments = "xen",
default = "vmm_ssh/vmm_ssh_xen.conf",
type = "xen" ]
TM_MAD = [
name = "tm_nfs",
executable = "one_tm",
arguments = "tm_nfs/tm_nfs.conf" ]
HM_MAD = [
executable = "one_hm" ]
VM_HOOK = [
name = "image",
on = "DONE",
command = "image.rb",
arguments = "$VMID" ]
VM_HOOK = [
name = "ebtables-start",
on = "running",
command = "/usr/share/one/hooks/ebtables-xen", # or ebtables-xen
arguments = "one-$VMID",
remote = "yes" ]
VM_HOOK = [
name = "ebtables-flush",
on = "done",
command = "/usr/share/one/hooks/ebtables-flush",
arguments = "",
remote = "yes" ]
# service oned stop
# service oned start
Add Xen Node
# onehost create one-02 im_xen vmm_xen tm_nfs
..... wait a minute...
# onehost list
ID NAME CLUSTER RVM TCPU FCPU ACPU TMEM FMEM STAT
0 one-02 default 1 200 196 100 2G 768M on
Prepare Virtual Network Template
# vi /home/oneadmin/template.vnet.public
NAME = "Public"
TYPE = "FIXED"
BRIDGE = "xenbr0"
LEASES = [ IP = "192.168.100.151" ]
LEASES = [ IP = "192.168.100.152" ]
LEASES = [ IP = "192.168.100.153" ]
LEASES = [ IP = "192.168.100.154" ]
LEASES = [ IP = "192.168.100.155" ]
LEASES = [ IP = "192.168.100.156" ]
LEASES = [ IP = "192.168.100.157" ]
LEASES = [ IP = "192.168.100.158" ]
LEASES = [ IP = "192.168.100.159" ]
LEASES = [ IP = "192.168.100.160" ]
Cretate Virtual Network
# onevnet create /home/oneadmin/template.vnet.public
# onevnet list
ID USER NAME TYPE BRIDGE P #LEASES
1 oneadmin Public Fixed xenbr0 N 1
Prepare Virtual Machine Template
# vi /home/oneadmin/template.vm.centos5_64
CPU = 1
MEMORY = 256
OS = [ bootloader = "/usr/bin/pygrub" ]
#DISK = [
# IMAGE ="Xen-CentOS5_64",
# target = "xvda" ]
DISK = [
source = "/home/oneadmin/centos5_64.xen.img",
target = "xvda",
readonly = "no" ]
DISK = [
type = "swap",
size = "1024",
target = "xvdb" ]
DISK = [
type = "fs",
format = "ext3",
size = "20480",
target = "xvdd" ]
NIC = [ NETWORK="Public" ]
GRAPHICS = [
type = "vnc",
listen = "127.0.0.1" ]
CONTEXT = [
files = "/home/oneadmin/init.sh /home/oneadmin/.ssh/id_rsa.pub /home/oneadmin/setup-network.sh",
root_pubkey = "id_rsa.pub",
VMID = "$VMID",
target = "xvdc" ]
Create Default Context Files
# vi /home/oneadmin/init.sh
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2009, Distributed Systems Architecture Group, Universidad #
# Complutense de Madrid (dsa-research.org) #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
CHK_FILE="/var/lib/one/.done_context"
/sbin/iptables -I INPUT -p icmp -j REJECT
if [ ! -e ${CHK_FILE} ]; then
## Random Password for root
# dd if=/dev/urandom count=128 2>/dev/null | md5sum | passwd --stdin root
MNT="/media/one"
if [ -f ${MNT}/context.sh ]; then
. ${MNT}/context.sh
fi
fdisk /dev/xvdb << EOF
n
p
1
t
82
w
EOF
if [ -b /dev/xvdb1 ]; then
mkswap /dev/xvdb1
echo "/dev/xvdb1 swap swap defaults 0 0" >> /etc/fstab
swapon -a
fi
#hostname $HOSTNAME
#sed -i "/HOSTNAME=/s/=.*$/=$HOSTNAME/" /etc/sysconfig/network
#if [ -n "$IP_PUBLIC" ]; then
# ifconfig eth0 $IP_PUBLIC
#fi
#if [ -n "$NETMASK" ]; then
# ifconfig eth0 netmask $NETMASK
#fi
if [ -f ${MNT}/$ROOT_PUBKEY ]; then
mkdir -p /root/.ssh
cat ${MNT}/$ROOT_PUBKEY >> /root/.ssh/authorized_keys
chmod -R 600 /root/.ssh/
fi
if [ -n "$USERNAME" ]; then
useradd $USERNAME
if [ -f ${MNT}/$USER_PUBKEY ]; then
mkdir -p /home/$USERNAME/.ssh/
cat ${MNT}/$USER_PUBKEY >> /home/$USERNAME/.ssh/authorized_keys
chown -R $USERNAME:$USERNAME /home/$USERNAME/.ssh
chmod -R 600 /home/$USERNAME/.ssh/authorized_keys
fi
fi
fi
### Network Setup
if [ -f ${MNT}/setup-network.sh ]; then
${MNT}/setup-network.sh
fi
if [ ! -e ${CHK_FILE} ]; then
### Run Deploy
if [ -f ${MNT}/deploy*/setup.sh ]; then
${MNT}/deploy*/setup.sh
fi
fi
## Etc Jobs
### VMS init Setup END
/sbin/iptables -D INPUT -p icmp -j REJECT
# vi /home/oneadmin/setup-network.sh
#!/bin/bash
#
# chkconfig: 2345 10 90
# description: network reconfigure
#
# -------------------------------------------------------------------------- #
# Copyright 2002-2009, Distributed Systems Architecture Group, Universidad #
# Complutense de Madrid (dsa-research.org) #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
# Gets IP address from a given MAC
mac2ip() {
mac=$1
let ip_a=0x`echo $mac | cut -d: -f 3`
let ip_b=0x`echo $mac | cut -d: -f 4`
let ip_c=0x`echo $mac | cut -d: -f 5`
let ip_d=0x`echo $mac | cut -d: -f 6`
ip="$ip_a.$ip_b.$ip_c.$ip_d"
echo $ip
}
# Gets the network part of an IP
get_network() {
IP=$1
echo $IP | cut -d'.' -f1,2,3
}
get_interfaces() {
IFCMD="/sbin/ifconfig -a"
$IFCMD | grep ^eth | sed 's/ *Link encap:Ethernet.*HWaddr /-/g'
}
get_dev() {
echo $1 | cut -d'-' -f 1
}
get_mac() {
echo $1 | cut -d'-' -f 2
}
gen_hosts() {
NETWORK=$1
echo "127.0.0.1 localhost"
for n in `seq -w 01 99`; do
n2=`echo $n | sed 's/^0*//'`
echo ${NETWORK}.$n2 cluster${n}
done
}
gen_exports() {
NETWORK=$1
echo "/images ${NETWORK}.0/255.255.255.0(rw,async,no_subtree_check)"
}
gen_hostname() {
MAC=$1
#NUM=`mac2ip $MAC | cut -d'.' -f4`
NUM=`mac2ip $MAC`
#NUM2=`echo 000000$NUM | sed 's/.*\(..\)/\1/'`
#echo cluster$NUM2
NUM2=`echo $NUM | sed 's/\./-/g'`
echo $NUM2
}
gen_interface() {
DEV_MAC=$1
DEV=`get_dev $DEV_MAC`
MAC=`get_mac $DEV_MAC`
IP=`mac2ip $MAC`
NETWORK=`get_network $IP`
cat <<EOT
DEVICE=$DEV
BOOTPROTO=none
HWADDR=$MAC
ONBOOT=yes
TYPE=Ethernet
NETMASK=255.255.255.0
IPADDR=$IP
EOT
if [ $DEV == "eth0" ]; then
echo " GATEWAY=$NETWORK.1"
fi
echo ""
}
IFACES=`get_interfaces`
for i in $IFACES; do
DEV=`get_dev $i`
gen_interface $i > /etc/sysconfig/network-scripts/ifcfg-${DEV}
done
# gen_hosts $NETWORK > /etc/hosts
# gen_exports $NETWORK > /etc/exports
# gen_hostname $MAC > /etc/hostname
#ifdown $DEV
#ifup $DEV
MNT="/media/one"
if [ -f ${MNT}/context.sh ]; then
. ${MNT}/context.sh
fi
HOSTNAME=ONE-${VMID}-`gen_hostname $MAC`
hostname $HOSTNAME
sed -i "/HOSTNAME=/s/=.*$/=$HOSTNAME/" /etc/sysconfig/network
if [ -n "$DNS" ]; then
for dns in $DNS
do
echo "nameserver $dns" > /etc/resolv.conf
done
else
echo "nameserver 168.126.63.1
nameserver 168.126.63.2" > /etc/resolv.conf
fi
service network restart
Creating VM and Connect SSH
Create VM
# onevm create /home/oneadmin/template.vm.centos5_64
..... wait a minute...
# onevm list
ID USER NAME STAT CPU MEM HOSTNAME TIME
37 oneadmin one-37 runn 0 256M one-02 07 04:40:21
# onevm show 37
VIRTUAL MACHINE 37 INFORMATION
ID : 37
NAME : one-37
STATE : ACTIVE
LCM_STATE : RUNNING
START TIME : 07/24 18:38:40
END TIME : -
DEPLOY ID: : one-37
VIRTUAL MACHINE MONITORING
NET_RX : 11
USED MEMORY : 262144
USED CPU : 0
NET_TX : 6887
VIRTUAL MACHINE TEMPLATE
CONTEXT=[
FILES=/home/oneadmin/init.sh /home/oneadmin/.ssh/id_rsa.pub /home/oneadmin/setup-network.sh,
ROOT_PUBKEY=id_rsa.pub,
TARGET=xvdc,
VMID=37 ]
CPU=1
DISK=[
CLONE=YES,
DISK_ID=0,
IMAGE=Xen-CentOS5_64,
IMAGE_ID=2,
READONLY=NO,
SAVE=NO,
SOURCE=/var/lib/one//images/334f3324668b29bd253c7d304e499576ede0b611,
TARGET=xvda,
TYPE=DISK ]
DISK=[
DISK_ID=1,
SIZE=1024,
TARGET=xvdb,
TYPE=swap ]
DISK=[
DISK_ID=2,
FORMAT=ext3,
SIZE=20480,
TARGET=xvdd,
TYPE=fs ]
GRAPHICS=[
LISTEN=127.0.0.1,
PORT=5937,
TYPE=vnc ]
MEMORY=256
NAME=one-37
NIC=[
BRIDGE=xenbr0,
IP=192.168.100.154,
MAC=02:00:c0:a8:64:9a,
NETWORK=Public,
NETWORK_ID=1 ]
OS=[
BOOTLOADER=/usr/bin/pygrub ]
VMID=37
Connect to VM's SSH
# ssh -i /home/oneadmin/.ssh/id_rsa root@192.168.100.154
Warning: Permanently added '192.168.100.154' (RSA) to the list of known hosts.
Last login: Sun Jul 31 22:35:14 2011 from 192.168.100.121
[root@ONE-37-192-168-100-154 ~]#
Some Info on VM
[root@ONE-37-192-168-100-154 ~]# cat /proc/cpuinfo
processor : 0
vendor_id : GenuineIntel
cpu family : 6
model : 23
model name : Intel(R) Core(TM)2 Quad CPU Q9550 @ 2.83GHz
stepping : 10
cpu MHz : 2826.250
cache size : 6144 KB
fpu : yes
fpu_exception : yes
cpuid level : 13
wp : yes
flags : fpu tsc msr pae cx8 apic cmov pat clflush acpi mmx fxsr sse sse2 ss syscall nx lm constant_tsc pni ssse3 cx16 sse4_1 lahf_lm
bogomips : 7097.80
clflush size : 64
cache_alignment : 64
address sizes : 40 bits physical, 48 bits virtual
power management:
[root@ONE-37-192-168-100-154 ~]# free -m
total used free shared buffers cached
Mem: 256 252 3 0 45 145
-/+ buffers/cache: 61 194
Swap: 1019 0 1019
[root@ONE-37-192-168-100-154 ~]# ifconfig
eth0 Link encap:Ethernet HWaddr 02:00:C0:A8:64:9A
inet addr:192.168.100.154 Bcast:192.168.100.255 Mask:255.255.255.0
UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1
RX packets:50726 errors:0 dropped:0 overruns:0 frame:0
TX packets:146 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:1000
RX bytes:7059149 (6.7 MiB) TX bytes:17642 (17.2 KiB)
lo Link encap:Local Loopback
inet addr:127.0.0.1 Mask:255.0.0.0
UP LOOPBACK RUNNING MTU:16436 Metric:1
RX packets:14 errors:0 dropped:0 overruns:0 frame:0
TX packets:14 errors:0 dropped:0 overruns:0 carrier:0
collisions:0 txqueuelen:0
RX bytes:700 (700.0 b) TX bytes:700 (700.0 b)
[root@ONE-37-192-168-100-154 ~]# route -n
Kernel IP routing table
Destination Gateway Genmask Flags Metric Ref Use Iface
192.168.100.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0
169.254.0.0 0.0.0.0 255.255.0.0 U 0 0 0 eth0
0.0.0.0 192.168.100.1 0.0.0.0 UG 0 0 0 eth0
[root@ONE-37-192-168-100-154 ~]# fdisk -l
Disk /dev/xvda: 1572 MB, 1572864000 bytes
255 heads, 63 sectors/track, 191 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/xvda1 * 1 13 104391 83 Linux
/dev/xvda2 14 191 1429785 83 Linux
Disk /dev/xvdb: 1073 MB, 1073741824 bytes
255 heads, 63 sectors/track, 130 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/xvdb1 1 130 1044193+ 82 Linux swap / Solaris
Disk /dev/xvdd: 21.4 GB, 21474836480 bytes
255 heads, 63 sectors/track, 2610 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/xvdd doesn't contain a valid partition table
Disk /dev/xvdc: 0 MB, 382976 bytes
255 heads, 63 sectors/track, 0 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Disk /dev/xvdc doesn't contain a valid partition table
[root@ONE-37-192-168-100-154 ~]# mount
/dev/xvda2 on / type ext3 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
/dev/xvda1 on /boot type ext3 (rw)
tmpfs on /dev/shm type tmpfs (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
/dev/xvdd on /mnt type ext3 (rw)
[root@ONE-37-192-168-100-154 ~]# df -Th
Filesystem Type Size Used Avail Use% Mounted on
/dev/xvda2 ext3 1.4G 995M 289M 78% /
/dev/xvda1 ext3 99M 14M 81M 14% /boot
tmpfs tmpfs 128M 0 128M 0% /dev/shm
[END]
