4 # For help on using kubespray with vagrant, check out docs/vagrant.md
8 Vagrant.require_version ">= 2.0.0"
10 CONFIG = File.join(File.dirname(__FILE__), ENV['KUBESPRAY_VAGRANT_CONFIG'] || 'vagrant/config.rb')
12 FLATCAR_URL_TEMPLATE = "https://%s.release.flatcar-linux.net/amd64-usr/current/flatcar_production_vagrant.json"
14 # Uniq disk UUID for libvirt
15 DISK_UUID = Time.now.utc.to_i
18 "flatcar-stable" => {box: "flatcar-stable", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["stable"]},
19 "flatcar-beta" => {box: "flatcar-beta", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["beta"]},
20 "flatcar-alpha" => {box: "flatcar-alpha", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["alpha"]},
21 "flatcar-edge" => {box: "flatcar-edge", user: "core", box_url: FLATCAR_URL_TEMPLATE % ["edge"]},
22 "ubuntu1604" => {box: "generic/ubuntu1604", user: "vagrant"},
23 "ubuntu1804" => {box: "generic/ubuntu1804", user: "vagrant"},
24 "ubuntu2004" => {box: "generic/ubuntu2004", user: "vagrant"},
25 "centos" => {box: "centos/7", user: "vagrant"},
26 "centos-bento" => {box: "bento/centos-7.6", user: "vagrant"},
27 "centos8" => {box: "centos/8", user: "vagrant"},
28 "centos8-bento" => {box: "bento/centos-8", user: "vagrant"},
29 "almalinux8" => {box: "almalinux/8", user: "vagrant"},
30 "almalinux8-bento" => {box: "bento/almalinux-8", user: "vagrant"},
31 "rockylinux8" => {box: "generic/rocky8", user: "vagrant"},
32 "fedora35" => {box: "fedora/35-cloud-base", user: "vagrant"},
33 "fedora36" => {box: "fedora/36-cloud-base", user: "vagrant"},
34 "opensuse" => {box: "opensuse/Leap-15.4.x86_64", user: "vagrant"},
35 "opensuse-tumbleweed" => {box: "opensuse/Tumbleweed.x86_64", user: "vagrant"},
36 "oraclelinux" => {box: "generic/oracle7", user: "vagrant"},
37 "oraclelinux8" => {box: "generic/oracle8", user: "vagrant"},
38 "rhel7" => {box: "generic/rhel7", user: "vagrant"},
39 "rhel8" => {box: "generic/rhel8", user: "vagrant"},
42 if File.exist?(CONFIG)
46 # Defaults for config options defined in CONFIG
48 $instance_name_prefix ||= "k8s"
52 $shared_folders ||= {}
53 $forwarded_ports ||= {}
54 $subnet ||= "172.18.8"
55 $subnet_ipv6 ||= "fd3c:b398:0698:0756"
57 $network_plugin ||= "flannel"
58 # Setting multi_networking to true will install Multus: https://github.com/intel/multus-cni
59 $multi_networking ||= "False"
60 $download_run_once ||= "True"
61 $download_force_cache ||= "False"
62 # The first three nodes are etcd servers
63 $etcd_instances ||= $num_instances
64 # The first two nodes are kube masters
65 $kube_master_instances ||= $num_instances == 1 ? $num_instances : ($num_instances - 1)
66 # All nodes are kube nodes
67 $kube_node_instances ||= $num_instances
68 # The following only works when using the libvirt provider
69 $kube_node_instances_with_disks ||= false
70 $kube_node_instances_with_disks_size ||= "20G"
71 $kube_node_instances_with_disks_number ||= 2
72 $override_disk_size ||= false
74 $local_path_provisioner_enabled ||= "False"
75 $local_path_provisioner_claim_root ||= "/opt/local-path-provisioner/"
76 $libvirt_nested ||= false
77 # boolean or string (e.g. "-vvv")
78 $ansible_verbosity ||= false
79 $ansible_tags ||= ENV['VAGRANT_ANSIBLE_TAGS'] || ""
81 $playbook ||= "cluster.yml"
85 $box = SUPPORTED_OS[$os][:box]
86 # if $inventory is not set, try to use example
87 $inventory = "inventory/sample" if ! $inventory
88 $inventory = File.absolute_path($inventory, File.dirname(__FILE__))
90 # if $inventory has a hosts.ini file use it, otherwise copy over
91 # vars etc to where vagrant expects dynamic inventory to be
92 if ! File.exist?(File.join(File.dirname($inventory), "hosts.ini"))
93 $vagrant_ansible = File.join(File.dirname(__FILE__), ".vagrant", "provisioners", "ansible")
94 FileUtils.mkdir_p($vagrant_ansible) if ! File.exist?($vagrant_ansible)
95 $vagrant_inventory = File.join($vagrant_ansible,"inventory")
96 FileUtils.rm_f($vagrant_inventory)
97 FileUtils.ln_s($inventory, $vagrant_inventory)
100 if Vagrant.has_plugin?("vagrant-proxyconf")
101 $no_proxy = ENV['NO_PROXY'] || ENV['no_proxy'] || "127.0.0.1,localhost"
102 (1..$num_instances).each do |i|
103 $no_proxy += ",#{$subnet}.#{i+100}"
107 Vagrant.configure("2") do |config|
110 if SUPPORTED_OS[$os].has_key? :box_url
111 config.vm.box_url = SUPPORTED_OS[$os][:box_url]
113 config.ssh.username = SUPPORTED_OS[$os][:user]
116 if Vagrant.has_plugin?("vagrant-vbguest") then
117 config.vbguest.auto_update = false
120 # always use Vagrants insecure key
121 config.ssh.insert_key = false
123 if ($override_disk_size)
124 unless Vagrant.has_plugin?("vagrant-disksize")
125 system "vagrant plugin install vagrant-disksize"
127 config.disksize.size = $disk_size
130 (1..$num_instances).each do |i|
131 config.vm.define vm_name = "%s-%01d" % [$instance_name_prefix, i] do |node|
133 node.vm.hostname = vm_name
135 if Vagrant.has_plugin?("vagrant-proxyconf")
136 node.proxy.http = ENV['HTTP_PROXY'] || ENV['http_proxy'] || ""
137 node.proxy.https = ENV['HTTPS_PROXY'] || ENV['https_proxy'] || ""
138 node.proxy.no_proxy = $no_proxy
141 ["vmware_fusion", "vmware_workstation"].each do |vmware|
142 node.vm.provider vmware do |v|
143 v.vmx['memsize'] = $vm_memory
144 v.vmx['numvcpus'] = $vm_cpus
148 node.vm.provider :virtualbox do |vb|
149 vb.memory = $vm_memory
152 vb.linked_clone = true
153 vb.customize ["modifyvm", :id, "--vram", "8"] # ubuntu defaults to 256 MB which is a waste of precious RAM
154 vb.customize ["modifyvm", :id, "--audio", "none"]
157 node.vm.provider :libvirt do |lv|
158 lv.nested = $libvirt_nested
159 lv.cpu_mode = "host-model"
160 lv.memory = $vm_memory
162 lv.default_prefix = 'kubespray'
163 # Fix kernel panic on fedora 28
165 lv.cpu_mode = "host-passthrough"
169 if $kube_node_instances_with_disks
171 driverletters = ('a'..'z').to_a
172 node.vm.provider :libvirt do |lv|
173 # always make /dev/sd{a/b/c} so that CI can ensure that
174 # virtualbox and libvirt will have the same devices to use for OSDs
175 (1..$kube_node_instances_with_disks_number).each do |d|
176 lv.storage :file, :device => "hd#{driverletters[d]}", :path => "disk-#{i}-#{d}-#{DISK_UUID}.disk", :size => $kube_node_instances_with_disks_size, :bus => "scsi"
181 if $expose_docker_tcp
182 node.vm.network "forwarded_port", guest: 2375, host: ($expose_docker_tcp + i - 1), auto_correct: true
185 $forwarded_ports.each do |guest, host|
186 node.vm.network "forwarded_port", guest: guest, host: host, auto_correct: true
189 if ["rhel7","rhel8"].include? $os
190 # Vagrant synced_folder rsync options cannot be used for RHEL boxes as Rsync package cannot
191 # be installed until the host is registered with a valid Red Hat support subscription
192 node.vm.synced_folder ".", "/vagrant", disabled: false
193 $shared_folders.each do |src, dst|
194 node.vm.synced_folder src, dst
197 node.vm.synced_folder ".", "/vagrant", disabled: false, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z'] , rsync__exclude: ['.git','venv']
198 $shared_folders.each do |src, dst|
199 node.vm.synced_folder src, dst, type: "rsync", rsync__args: ['--verbose', '--archive', '--delete', '-z']
203 ip = "#{$subnet}.#{i+100}"
204 node.vm.network :private_network, ip: ip,
205 :libvirt__guest_ipv6 => 'yes',
206 :libvirt__ipv6_address => "#{$subnet_ipv6}::#{i+100}",
207 :libvirt__ipv6_prefix => "64",
208 :libvirt__forward_mode => "none",
209 :libvirt__dhcp_enabled => false
211 # Disable swap for each vm
212 node.vm.provision "shell", inline: "swapoff -a"
214 # ubuntu1804 and ubuntu2004 have IPv6 explicitly disabled. This undoes that.
215 if ["ubuntu1804", "ubuntu2004"].include? $os
216 node.vm.provision "shell", inline: "rm -f /etc/modprobe.d/local.conf"
217 node.vm.provision "shell", inline: "sed -i '/net.ipv6.conf.all.disable_ipv6/d' /etc/sysctl.d/99-sysctl.conf /etc/sysctl.conf"
220 # Disable firewalld on oraclelinux/redhat vms
221 if ["oraclelinux","oraclelinux8","rhel7","rhel8"].include? $os
222 node.vm.provision "shell", inline: "systemctl stop firewalld; systemctl disable firewalld"
225 host_vars[vm_name] = {
227 "flannel_interface": "eth1",
228 "kube_network_plugin": $network_plugin,
229 "kube_network_plugin_multus": $multi_networking,
230 "download_run_once": $download_run_once,
231 "download_localhost": "False",
232 "download_cache_dir": ENV['HOME'] + "/kubespray_cache",
233 # Make kubespray cache even when download_run_once is false
234 "download_force_cache": $download_force_cache,
235 # Keeping the cache on the nodes can improve provisioning speed while debugging kubespray
236 "download_keep_remote_cache": "False",
237 "docker_rpm_keepcache": "1",
238 # These two settings will put kubectl and admin.config in $inventory/artifacts
239 "kubeconfig_localhost": "True",
240 "kubectl_localhost": "True",
241 "local_path_provisioner_enabled": "#{$local_path_provisioner_enabled}",
242 "local_path_provisioner_claim_root": "#{$local_path_provisioner_claim_root}",
243 "ansible_ssh_user": SUPPORTED_OS[$os][:user]
246 # Only execute the Ansible provisioner once, when all the machines are up and ready.
247 # And limit the action to gathering facts, the full playbook is going to be ran by testcases_run.sh
248 if i == $num_instances
249 node.vm.provision "ansible" do |ansible|
250 ansible.playbook = $playbook
251 ansible.verbose = $ansible_verbosity
252 $ansible_inventory_path = File.join( $inventory, "hosts.ini")
253 if File.exist?($ansible_inventory_path)
254 ansible.inventory_path = $ansible_inventory_path
256 ansible.become = true
257 ansible.limit = "all,localhost"
258 ansible.host_key_checking = false
259 ansible.raw_arguments = ["--forks=#{$num_instances}", "--flush-cache", "-e ansible_become_pass=vagrant"]
260 ansible.host_vars = host_vars
261 if $ansible_tags != ""
262 ansible.tags = [$ansible_tags]
265 "etcd" => ["#{$instance_name_prefix}-[1:#{$etcd_instances}]"],
266 "kube_control_plane" => ["#{$instance_name_prefix}-[1:#{$kube_master_instances}]"],
267 "kube_node" => ["#{$instance_name_prefix}-[1:#{$kube_node_instances}]"],
268 "k8s_cluster:children" => ["kube_control_plane", "kube_node"],