Docker Swarm: Native Docker Clustering [2016, ENG] > Module 5: Securing your Swarm Cluster


Native Docker Clustering


Подготовил с помощью следующего vagrant скрипта следующее:

192.168.56.101 client
192.168.56.102 ca
192.168.56.103 manager1
192.168.56.104 manager2
192.168.56.105 manager3
192.168.56.106 node1
192.168.56.107 node2
192.168.56.108 node3


CREATE CA


$ vagrant ssh ca
$ sudo su -

# mkdir -p /home/debian/certs/
# cd /home/debian/certs/

# openssl genrsa -out ca-key.pem 2048

# ls
ca-key.pem


# openssl req -config /usr/lib/ssl/openssl.cnf -new -key ca-key.pem -x509 -days 1825 -out ca-cert.pem

Country Name (2 letter code) [AU]:UK
State or Province Name (full name) [Some-State]:CH
Locality Name (eg, city) []:Sunderland
Organization Name (eg, company) [Internet Widgits Pty Ltd]:
Organizational Unit Name (eg, section) []:
Common Name (e.g. server FQDN or YOUR name) []:
Email Address []:

# ls
ca-cert.pem  ca-key.pem


CREATE MANAGER AND NODE KEYS

# openssl genrsa -out manager1-key.pem 2048
# openssl req -subj "/CN=manager1" -new -key manager1-key.pem -out manager1.csr
# echo subjectAltName = IP:192.168.56.103,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in manager1.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager1-cert.pem -extfile extfile.cnf


# openssl genrsa -out manager2-key.pem 2048
# openssl req -subj "/CN=manager2" -new -key manager2-key.pem -out manager2.csr
# echo subjectAltName = IP:192.168.56.104,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in manager2.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager2-cert.pem -extfile extfile.cnf


# openssl genrsa -out manager3-key.pem 2048
# openssl req -subj "/CN=manager3" -new -key manager3-key.pem -out manager3.csr
# echo subjectAltName = IP:192.168.56.105,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in manager3.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out manager3-cert.pem -extfile extfile.cnf


# openssl genrsa -out node1-key.pem 2048
# openssl req -subj "/CN=node1" -new -key node1-key.pem -out node1.csr
# echo subjectAltName = IP:192.168.56.106,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in node1.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node1-cert.pem -extfile extfile.cnf


# openssl genrsa -out node2-key.pem 2048
# openssl req -subj "/CN=node2" -new -key node2-key.pem -out node2.csr
# echo subjectAltName = IP:192.168.56.107,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in node2.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node2-cert.pem -extfile extfile.cnf


# openssl genrsa -out node3-key.pem 2048
# openssl req -subj "/CN=node3" -new -key node3-key.pem -out node3.csr
# echo subjectAltName = IP:192.168.56.108,IP:127.0.0.1 > extfile.cnf
# openssl x509 -req -days 365 -in node3.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out node3-cert.pem -extfile extfile.cnf


CREATE CLIENT KEYS

# openssl genrsa -out client-key.pem 2048
# openssl req -subj "/CN=client" -new -key client-key.pem -out client.csr
# echo extendedKeyUsage = clientAuth > extfile.cnf
# openssl x509 -req -days 365 -in client.csr -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem -extfile extfile.cnf


# ls -l
total 100
-rw-r--r-- 1 root root 1261 Feb 12 11:50 ca-cert.pem
-rw-r--r-- 1 root root   17 Feb 12 12:06 ca-cert.srl
-rw-r--r-- 1 root root 1675 Feb 12 11:49 ca-key.pem
-rw-r--r-- 1 root root 1099 Feb 12 12:02 client-cert.pem
-rw-r--r-- 1 root root  887 Feb 12 12:02 client.csr
-rw-r--r-- 1 root root 1679 Feb 12 12:01 client-key.pem
-rw-r--r-- 1 root root   48 Feb 12 12:06 extfile.cnf
-rw-r--r-- 1 root root 1103 Feb 12 12:06 manager1-cert.pem
-rw-r--r-- 1 root root  891 Feb 12 12:04 manager1.csr
-rw-r--r-- 1 root root 1679 Feb 12 12:04 manager1-key.pem
-rw-r--r-- 1 root root 1103 Feb 12 11:58 manager2-cert.pem
-rw-r--r-- 1 root root  891 Feb 12 11:58 manager2.csr
-rw-r--r-- 1 root root 1679 Feb 12 11:58 manager2-key.pem
-rw-r--r-- 1 root root 1103 Feb 12 11:59 manager3-cert.pem
-rw-r--r-- 1 root root  891 Feb 12 11:58 manager3.csr
-rw-r--r-- 1 root root 1675 Feb 12 11:58 manager3-key.pem
-rw-r--r-- 1 root root 1099 Feb 12 12:00 node1-cert.pem
-rw-r--r-- 1 root root  887 Feb 12 12:00 node1.csr
-rw-r--r-- 1 root root 1679 Feb 12 12:00 node1-key.pem
-rw-r--r-- 1 root root 1099 Feb 12 12:01 node2-cert.pem
-rw-r--r-- 1 root root  887 Feb 12 12:00 node2.csr
-rw-r--r-- 1 root root 1679 Feb 12 12:00 node2-key.pem
-rw-r--r-- 1 root root 1099 Feb 12 12:01 node3-cert.pem
-rw-r--r-- 1 root root  887 Feb 12 12:01 node3.csr
-rw-r--r-- 1 root root 1675 Feb 12 12:01 node3-key.pem


COPY KEYS

На нодах сначала нужно создать .docker

mkdir .docker
chmod 777 .docker/


# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem


Manager1

$ vagrant ssh manager1
$ sudo su -

$ adduser debian
# su - debian

$ mkdir ~/.docker
$ chmod 777 ~/.docker/

c CA

# scp ./ca-cert.pem [email protected]:/home/debian/.docker/ca.pem

# scp ./manager1-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./manager1-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Manager2

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./manager2-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./manager2-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Manager3

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./manager3-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./manager3-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Node1

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./node1-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./node1-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Node2

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./node2-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./node2-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Node3

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./node3-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./node3-key.pem [email protected]:/home/ubuntu/.docker/key.pem


Client

# scp ./ca-cert.pem [email protected]:/home/ubuntu/.docker/ca.pem

# scp ./client-cert.pem [email protected]:/home/ubuntu/.docker/cert.pem

# scp ./client-key.pem [email protected]:/home/ubuntu/.docker/key.pem


DOCKER DAEMON restarts

На всех нодах

# vim /etc/default/docker

Заменили на (было закомментировано):

DOCKER_OPTS="-H tcp://0.0.0.0:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem"


# service docker start
# service docker status

# ps -elf | grep docker


START NEW CONSUL SERVERS


MANAGER1

# docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul1 --name consul1 -v /mnt:/data     -p 10.0.1.5:8300:8300     -p 10.0.1.5:8301:8301     -p 10.0.1.5:8301:8301/udp     -p 10.0.1.5:8302:8302     -p 10.0.1.5:8302:8302/udp     -p 10.0.1.5:8400:8400     -p 10.0.1.5:8500:8500     -p 172.17.0.1:53:53/udp     progrium/consul -server -advertise 10.0.1.5 -join 10.0.2.5


MANAGER2

# docker -H tcp://manager2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul2 --name consul2 -v /mnt:/data     -p 10.0.2.5:8300:8300     -p 10.0.2.5:8301:8301     -p 10.0.2.5:8301:8301/udp     -p 10.0.2.5:8302:8302     -p 10.0.2.5:8302:8302/udp     -p 10.0.2.5:8400:8400     -p 10.0.2.5:8500:8500     -p 172.17.0.1:53:53/udp     progrium/consul -server -advertise 10.0.1.5 -join 10.0.1.5


MANAGER3

# docker -H tcp://manager3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul3 --name consul3 -v /mnt:/data     -p 10.0.3.5:8300:8300     -p 10.0.3.5:8301:8301     -p 10.0.3.5:8301:8301/udp     -p 10.0.3.5:8302:8302     -p 10.0.3.5:8302:8302/udp     -p 10.0.3.5:8400:8400     -p 10.0.3.5:8500:8500     -p 172.17.0.1:53:53/udp     progrium/consul -server -advertise 10.0.1.5 -join 10.0.1.5


# docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr1 --name mgr1 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.1.5:2376 consul://10.0.1.5:8500/


START CONSUL CLIENTS


NODE1

# docker -H tcp://node1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt1 --name consul-agt1 \
-p 8300:8300 \
-p 8301:8301 -p 8301:8301/udp \
-p 8302:8302 -p 8302:8302/udp \
-p 8400:8400 \
-p 8500:8500 \
-p 8600:8600/udp \
progrium/consul -rejoin -advertise 10.0.4.5 -join 10.0.1.5


NODE2

# docker -H tcp://node2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt2 --name consul-agt2 \
-p 8300:8300 \
-p 8301:8301 -p 8301:8301/udp \
-p 8302:8302 -p 8302:8302/udp \
-p 8400:8400 \
-p 8500:8500 \
-p 8600:8600/udp \
progrium/consul -rejoin -advertise 10.0.5.5 -join 10.0.1.5


NODE3

# docker -H tcp://node3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -d -h consul-agt3 --name consul-agt3 \
-p 8300:8300 \
-p 8301:8301 -p 8301:8301/udp \
-p 8302:8302 -p 8302:8302/udp \
-p 8400:8400 \
-p 8500:8500 \
-p 8600:8600/udp \
progrium/consul -rejoin -advertise 10.0.6.5 -join 10.0.1.5


START SWARM MANAGERS


MNAGER1

# docker -H tcp://manager1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr1 --name mgr1 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.1.5:2376 consul://10.0.1.5:8500/


MANAGER2

# docker -H tcp://manager2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr2 --name mgr2 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.2.5:2376 consul://10.0.2.5:8500/


MANAGER3

# docker -H tcp://manager3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run --restart=unless-stopped -h mgr3 --name mgr3 -d -p 3376:2376 -v /home/ubuntu/.docker:/certs:ro swarm manage --tlsverify --tlscacert=/certs/ca.pem --tlscert=/certs/cert.pem --tlskey=/certs/key.pem --host=0.0.0.0:2376 --replication --advertise 10.0.3.5:2376 consul://10.0.3.5:8500/


START SWARM JOIN CONTAIENRS ON EACH NODE


NODE1

# docker -H tcp://node1:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.4.5:2376 consul://10.0.4.5:8500/


NODE2

# docker -H tcp://node2:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.5.5:2376 consul://10.0.5.5:8500/


NODE3

# docker -H tcp://node3:2376 --tlsverify --tlscacert=/home/ubuntu/.docker/ca.pem --tlscert=/home/ubuntu/.docker/cert.pem --tlskey=/home/ubuntu/.docker/key.pem run -d -h join --name join swarm join --advertise=10.0.6.5:2376 consul://10.0.6.5:8500/


Client

$ export DOCKER_HOST=manager1:3376
$ docker version

$ export DOCKER_TLS_VERIFY=1
$ export DOCKER_CERT_PATH=/home/ubuntu/.docker

$ ls -l .docker/
(3 сертификата)

$ docker version
(получаем данные о клиенте и сервере)


Filtering and Scheduling

$ docker run -dit ubuntu /bin/bash
$ docker run -dit ubuntu /bin/bash
$ docker run -dit ubuntu /bin/bash

$ docker ps

$ for i in {1..30}; do /bin/bash -c "docker run -dit ubuntu /bin/bash"; done


Scheduling with RAM Reservations

$ docker run -d -m 900m nginx
$ docker run -d -m 900m nginx

$ docker info


Affinity Filters

$ docker run -d --name c1 nginx
$ docker run -d --name c2 -e affinity:container==c1 nginx

$ docker run -d --name c3 -e affinity:container!=c1 nginx