Setting up Docker on a Warewulf provisioned cluster is very similar to the multihost setup. For this example we are going to use version 1.2 of Docker

Note that this example uses
/home/test <yourproxy:port>

which would all need to be replaced with site specific settings.

1. Setting up a head node for Docker First install lxc and docker

    zypper -n in lxc
    zypper -n in docker

2. Increase some docker limits

    % vi /usr/lib/systemd/system/docker.service # add the following to the Service section
    LimitNOFILE=1048576
    LimitNPROC=1048576

3. Disable Docker's automatic bridge device and setup proxy We need to disable docker from managing the bridge device since we want to manage it by hand, and we will need to use lxc instead of libcontainer to have control over setting our own container IP addresses. Configure this in /etc/sysconfig/docker.

    % perl -pi -e 's/^DOCKER_OPTS/#DOCKER_OPTS/' /etc/sysconfig/docker
    % echo "DOCKER_OPTS=\"-b=none -e=lxc\"" >> /etc/sysconfig/docker
    % echo "PROXY_ENABLED=\"yes\"" >> /etc/sysconfig/docker
    % echo "HTTP_PROXY=\"http://<yourproxy:port>/\"" >> /etc/sysconfig/docker
    % echo "HTTPS_PROXY=\"https://<yourproxy:port>/\"" >> /etc/sysconfig/docker
    % echo "FTP_PROXY=\"http://<yourproxy:port>/\"" >> /etc/sysconfig/docker
    % echo "NO_PROXY=\"localhost, 127.0.0.1, .<yourdomain>\"" >> /etc/sysconfig/docker
    % systemctl restart docker

4. Setup our own bridge device

    % cat <<EOF > /etc/sysconfig/network/ifcfg-eth0
    % DEVICE=dockerbridge0
    % TYPE=Bridge
    % BOOTPROTO=none
    % ONBOOT=yes
    % IPV6INIT=no
    % IPV6_AUTOCONF=no
    % STP=no
    % EOF
 
    % cat <<EOF > /etc/sysconfig/network/ifcfg-dockerbridge0
    % DEVICE=eth0
    % BOOTPROTO=dhcp
    % HWADDR=
    % ONBOOT=yes
    % NETMASK=
    % GATEWAY=
    % BRIDGE=dockerbridge0
    % BRIDGE_PORTS=eth0
    % EOF

5. Drop the previous address binding

    % ip addr flush eth0
    % ifup eth0
    % brctl addbr dockerbridge0
    % brctl addif dockerbridge0 eth0
    % ifup dockerbridge0

6. Set up provisioning of nodes with support for Docker We need to enable the provisioned initrd to have support for Docker usage of bridges, for this we add the bridge kernel module to it. In Warewulf it will look something like this:

    % echo "# Added support for network bridges for Docker" >> /etc/warewulf/bootstrap.conf
    % echo "drivers += kernel/net/bridge" >> /etc/warewulf/bootstrap.conf
    % echo "modprobe += bridge" >> /etc/warewulf/bootstrap.conf

7. Add the bridge-utils and docker packages to the vnfs image

    % zypper -n --root=<chroot> install bridge-utils
    % zypper -n --root=<chroot> install lxc
    % zypper -n --root=<chroot> install docker
    % zypper -n --root=<chroot> install bridge-utils
    % zypper -n --root=<chroot> install -t pattern apparmor

8. Setup our own bridge device

    % cat <<EOF > <chroot>/etc/sysconfig/network/ifcfg-eth0
    % DEVICE=dockerbridge0
    % TYPE=Bridge
    % BOOTPROTO=none
    % ONBOOT=yes
    % IPV6INIT=no
    % IPV6_AUTOCONF=no
    % STP=no
    % EOF
 
    % cat <<EOF > <chroot>/etc/sysconfig/network/ifcfg-dockerbridge0
    % DEVICE=eth0
    % BOOTPROTO=dhcp
    % HWADDR=
    % ONBOOT=yes
    % NETMASK=
    % GATEWAY=
    % BRIDGE=dockerbridge0
    % BRIDGE_PORTS=eth0
    % EOF

9. Disable Docker's automatic bridge device

    % perl -pi -e 's/^DOCKER_OPTS/#DOCKER_OPTS/' <chroot>/etc/sysconfig/docker
    % echo "DOCKER_OPTS=\"-b=none -e=lxc\"" >> <chroot>/etc/sysconfig/docker

10. Drop the previous address binding We use etc/rc.d/after.local to add the Docker bridge. Note that this is not very friendly to systemd startup. We also add an optional route to the headnode if packet forwarding is needed.

    % echo "# Drop the previous address binding" >> <chroot>/etc/rc.d/after.local
    % echo "ip addr flush eth0" >> <chroot>/etc/rc.d/after.local
    % echo "ifup eth0" >> <chroot>/etc/rc.d/after.local
    % echo "brctl addbr dockerbridge0" >> <chroot>/etc/rc.d/after.local
    % echo "brctl addif dockerbridge0 eth0" >> <chroot>/etc/rc.d/after.local
    % echo "ifup dockerbridge0" >> <chroot>/etc/rc.d/after.local
    % echo "route add default gw 192.168.0.1 dockerbridge0" >> <chroot>/etc/rc.d/after.local
    % echo "service docker restart" >> <chroot>/etc/rc.d/after.local

11. Use the new bridge for provisioning

    % perl -pi -e 's/^network device = eth0/network device = dockerbridge0/' /etc/warewulf/provision.conf
    % perl -pi -e 's/DHCPD_INTERFACE="eth0"/DHCPD_INTERFACE="dockerbridge0"/' /etc/sysconfig/dhcpd

12. Rebuild the bootstrap and the vnfs

    % wwbootstrap `uname -r`
    % wwvnfs -y --chroot <chroot>

We use the following IP address scheme to create the hosts file:

    192.168.254.<1-32> -- used for AG nodes
    192.168.254.255 -- used for Master / Scheduler node
    192.168.254.254 -- used for DB node
    192.168.254.200 -- login node
    192.168.254.201 -- login node2
    192.168.255.<1-32> -- used for host addresses
    192.168.<1-32>.<0-255> -- used for CNs

1. Create the hosts file We use the following bash script to create the hosts file that will be sent to Docker. Note that this uses the orcm-site.xml 3.0 support.

    % echo "192.168.254.250 db" > orcm_hosts.txta
    % echo "192.168.254.255 master01" >> orcm_hosts.txt
    % echo "192.168.254.1 agg01" >> orcm_hosts.txt
    % echo "192.168.254.2 agg02" >> orcm_hosts.txt
    % echo "192.168.254.3 agg03" >> orcm_hosts.txt
    % echo "192.168.254.4 agg04" >> orcm_hosts.txt
    % n=0; for i in `seq -w 0 0255`; do echo "192.168.1.$n node$i" ; n=$((n+1)); done >> orcm_hosts.txt
    % n=0; for i in `seq -w 0256 0511`; do echo "192.168.2.$n node$i" ; n=$((n+1)); done >> orcm_hosts.txt
    % n=0; for i in `seq -w 0512 0767`; do echo "192.168.3.$n node$i" ; n=$((n+1)); done >> orcm_hosts.txt
    % n=0; for i in `seq -w 0768 1023`; do echo "192.168.4.$n node$i" ; n=$((n+1)); done >> orcm_hosts.txt
 
    % cat <<EOF > orcm-site.xml
    % <?xml version="1.0" encoding="UTF-8" ?>
    % <configuration>
    %    <version>3.0</version>
    %    <role>RECORD</role>
    %    <junction>
    %        <type>cluster</type>
    %        <name>master3</name>
    %        <junction>
    %            <type>row</type>
    %            <name>row1</name>
    % EOF
    % n=0; m=255; for i in `seq -w 01 04`; do cat <<EOF >> orcm-site.xml ;  n=$((n+256)); m=$((n+255)); done
    %            <junction>
    %                <type>rack</type>
    %                <name>agg$i</name>
    %                <controller>
    %                    <host>agg$i</host>
    %                    <port>55805</port>
    %                    <aggregator>yes</aggregator>
    %                </controller>
    %                <junction>
    %                    <type>node</type>
    %                    <name>node[4:$n-$m]</name>
    %                    <controller>
    %                        <host>@</host>
    %                        <port>55805</port>
    %                        <aggregator>no</aggregator>
    %                    </controller>
    %                </junction>
    %            </junction>
    % EOF
    % cat <<EOF >> orcm-site.xml
    %        </junction>
    %    </junction>
    %    <scheduler>
    %        <shost>master01</shost>
    %        <port>55820</port>
    %    </scheduler>
    % </configuration>
    % EOF

2. Create the launch scripts for the master and the aggregator daemons (AGs)

    # Master
    % echo "/usr/bin/docker run -d --name master01 -h master01 -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.254.255/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker image> /opt/openrcm/bin/orcmsched " >> master_docker_start.txt;
 
    # agg01
    % echo "/usr/bin/docker run -d --name agg01 -h agg01 -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.254.1/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker image> /opt/openrcm/bin/orcmd " > agg01_docker_start.txt;
    # agg02
    % echo "/usr/bin/docker run -d --name agg02 -h agg02 -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.254.2/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker image> /opt/openrcm/bin/orcmd " > agg02_docker_start.txt;
    # agg03
    % echo "/usr/bin/docker run -d --name agg03 -h agg03 -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.254.3/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker image> /opt/openrcm/bin/orcmd " > agg03_docker_start.txt;
    # agg04
    % echo "/usr/bin/docker run -d --name agg04 -h agg04 -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.254.4/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker image> /opt/openrcm/bin/orcmd " > agg04_docker_start.txt;

3. Create the launch scripts for the Compute Node Deamons (CNs)

    % # CN for node0-255
    % rm c1_docker_start.txt
    % n=0; for i in `seq -w 0 0255`; do
    % echo "/usr/bin/docker run -d --name node$i -h node$i -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.1.$n/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker_image> /opt/openrcm/bin/orcmd " >> c1_docker_start.txt;
    % n=$((n+1));
    % done
 
    % # CN for node256-511
    % rm c2_docker_start.txt
    % n=0; for i in `seq -w 0256 0511`; do
    % echo "/usr/bin/docker run -d --name node$i -h node$i -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.2.$n/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker_image> /opt/openrcm/bin/orcmd " >> c2_docker_start.txt
    % n=$((n+1));
    % done
 
    % # CN for node512-767
    % rm c3_docker_start.txt
    % n=0; for i in `seq -w 0512 0767`; do
    % echo "/usr/bin/docker run -d --name node$i -h node$i -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml \
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.3.$n/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker_image> /opt/openrcm/bin/orcmd " >> c3_docker_start.txt
    % n=$((n+1));
    % done
 
    % # CN for node768-1023
    % rm c4_docker_start.txt
    % n=0; for i in `seq -w 0768 1023`; do
    % echo "/usr/bin/docker run -d --name node$i -h node$i -v /home/test/orcm_hosts.txt:/etc/hosts \
        -v /home/test/openmpi-mca-params.conf:/opt/openrcm/etc/openmpi-mca-params.conf \
        -v /home/test/orcm-site.xml:/opt/openrcm/etc/orcm-site.xml\
        --lxc-conf=\"lxc.network.type = veth\" --lxc-conf=\"lxc.network.ipv4 = 192.168.4.$n/16\" \
        --lxc-conf=\"lxc.network.link = dockerbridge0\" --lxc-conf=\"lxc.network.name = eth0\" \
        --lxc-conf=\"lxc.network.flags = up\" <docker_image> /opt/openrcm/bin/orcmd " >> c4_docker_start.txt
    % n=$((n+1));
    % done

4. Launch the master and AG nodes

    % # scheduler running on head node
    % bash master_docker_start.txt
    % # AGs 
    % pdsh -w c[01-04] 'bash /home/test/agg0$((%n + 1))_docker_start.txt'

5. Launch the CN nodes

    % # Docker CNs 
    % pdsh -w c[05-08] 'bash /home/test/c$((%n + 1))_docker_start.txt'

6. Stopping and removing docker containers

    % pdsh -w c[01-04]  'docker stop $(docker ps -a -q)' 
    % pdsh -w c[01-04]  'docker rm $(docker ps -a -q)' 
    % pdsh -w c[05-08]  'docker stop $(docker ps -a -q)' 
    % pdsh -w c[05-08]  'docker rm $(docker ps -a -q)' 
    % docker stop $(docker ps -a -q)
    % docker rm $(docker ps -a -q)