diff --git a/.gitignore b/.gitignore index d07c135..6261880 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,11 @@ .apt-mirror-config .customDocker.sh +.customVM.sh .proxy-config .vagrant DEADJOE -Vagrantfile /files/log /files/kaz/download /files/kaz/git /files/kaz/log +/files/customVM.sh diff --git a/README.md b/README.md index 8753d81..eebd555 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,6 @@ Nous utilisons : Vous avez besoin de [vagrant](https://www.vagrantup.com/), [VirtualBox](https://www.virtualbox.org/) et éventuellement git. -UDP/53 ne doit pas être filtré depuis votre poste (par un firewall d'entreprise par exemple). Pour tester: -```bash -# dig @80.67.169.12 www.kaz.bzh -``` ## Installation @@ -34,18 +30,14 @@ git clone git+ssh://git@git.kaz.bzh:2202/KAZ/kaz-vagrant.git # pour contribuer cd kaz-vagrant/ git switch develop-snster # dans les 2 cas ``` -* Personalisez votre simulateur avec la commande (au besoin ajustez la mémoire et les cpus utilisés dans Vagrantfile) : -```bash -vagrant plugin install vagrant-disksize -vagrant plugin install vagrant-vbguest -./init.sh # vous pouvez laisser les choix par défaut -``` +* (Optionnel) Ajustez éventuellement la mémoire et les cpus utilisés dans Vagrantfile (par défaut 4GB et 2 vCPUs) + * Pour créer tout l'univers Kaz il faut se placer dans le répertoire et lancer la commande : ```bash vagrant up ``` -Cette étape peut-être (très) longue. Notamment, la construction de kaz-prod se fait dans un conteneur LXC, dans lequel les overlays docker passent par un filesystem FUSE beaucoup plus lent qu'en natif... +Cette étape peut-être (très) longue. Notamment, la construction de kaz-prod se fait dans un conteneur LXC, dans lequel les overlays docker passent par un filesystem plus lent qu'en natif... Comptez entre 40 minutes et quelques heures, selon la connexion réseau et les performances de la machine. ## Mise au point diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..9162983 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,50 @@ +# coding: utf-8 +# -*- mode: ruby -*- +# vi: set ft=ruby : + +unless Vagrant.has_plugin?("vagrant-disksize") + raise Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Please install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'" +end + +# All Vagrant configuration is done below. The "2" in Vagrant.configure +# configures the configuration version (we support older styles for +# backwards compatibility). Please don't change it unless you know what +# you're doing. +Vagrant.configure("2") do |config| + + required_plugins = %w( vagrant-vbguest vagrant-disksize ) + _retry = false + required_plugins.each do |plugin| + unless Vagrant.has_plugin? plugin + system "vagrant plugin install #{plugin}" + _retry=true + end + end + + if (_retry) + exec "vagrant " + ARGV.join(' ') + end + + config.vm.box = "debian/bullseye64" + config.vm.hostname = 'kaz-vm' + config.disksize.size = '32GB' + + config.vm.provider "virtualbox" do |vb| + vb.memory = "4096" + vb.cpus = "2" + vb.name = "kaz-vm" + vb.customize ["modifyvm", :id, "--vram", "64", "--clipboard-mode", "bidirectional", '--graphicscontroller', 'vmsvga', '--natnet1', '192.168.64.0/24'] + vb.gui = true + end + + + + #permet d'avoir un répertoire partagé entre la VM et le host + config.vm.synced_folder "/tmp/", "/tmp_host" + config.vm.synced_folder "files/", "/root/kaz-vagrant" + + config.vm.provision "shell" do |s| + s.inline = "/vagrant/files/vm-provision.sh" + s.env = {"KAZGUARD" => "true", "HOSTLANG" => ENV['LANG'], "NOKAZ" => ENV['NOKAZ'], "KAZBRANCH" => ENV['KAZBRANCH']} + end +end diff --git a/Vagrantfile.dist b/Vagrantfile.dist deleted file mode 100644 index 69e6925..0000000 --- a/Vagrantfile.dist +++ /dev/null @@ -1,95 +0,0 @@ -# coding: utf-8 -# -*- mode: ruby -*- -# vi: set ft=ruby : - -unless Vagrant.has_plugin?("vagrant-disksize") - raise Vagrant::Errors::VagrantError.new, "vagrant-disksize plugin is missing. Please install it using 'vagrant plugin install vagrant-disksize' and rerun 'vagrant up'" -end - -# All Vagrant configuration is done below. The "2" in Vagrant.configure -# configures the configuration version (we support older styles for -# backwards compatibility). Please don't change it unless you know what -# you're doing. -Vagrant.configure("2") do |config| - - required_plugins = %w( vagrant-vbguest vagrant-disksize ) - _retry = false - required_plugins.each do |plugin| - unless Vagrant.has_plugin? plugin - system "vagrant plugin install #{plugin}" - _retry=true - end - end - - if (_retry) - exec "vagrant " + ARGV.join(' ') - end - - config.vm.box = "debian/bullseye64" - config.vm.hostname = 'kaz-vm' - config.disksize.size = '32GB' - - # Disable automatic box update checking. If you disable this, then - # boxes will only be checked for updates when the user runs - # `vagrant box outdated`. This is not recommended. - # config.vm.box_check_update = false - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # config.vm.network "forwarded_port", guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network "private_network", ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network "public_network" - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - # config.vm.synced_folder "data", "/vagrant_data" - # config.vm.synced_folder "..", "/root/mi-lxc", create:true, type:"rsync", - # rsync__exclude: [".git/", "zzlocal/", "vagrant/"] - - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - config.vm.provider "virtualbox" do |vb| - # # Display the VirtualBox GUI when booting the machine - # vb.gui = true - # - # # Customize the amount of memory on the VM: - vb.memory = "4096" - vb.cpus="2" - vb.name = "kaz-vm" - vb.customize ["modifyvm", :id, "--vram", "64", "--clipboard-mode", "bidirectional", '--graphicscontroller', 'vmsvga', '--natnet1', '192.168.64.0/24'] - vb.gui = true - - end - - # Define a Vagrant Push strategy for pushing to Atlas. Other push strategies - # such as FTP and Heroku are also available. See the documentation at - # https://docs.vagrantup.com/v2/push/atlas.html for more information. - # config.push.define "atlas" do |push| - # push.app = "YOUR_ATLAS_USERNAME/YOUR_APPLICATION_NAME" - # end - - # Enable provisioning with a shell script. Additional provisioners such as - # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the - # documentation for more information about their specific syntax and use. - - #permet d'avoir un répertoire partagé entre la VM et le host - config.vm.synced_folder "/tmp/", "/tmp_host" - config.vm.synced_folder "files/", "/root/kaz-vagrant" - - config.vm.provision "shell" do |s| - s.inline = "/vagrant/files/vm-provision.sh" - s.env = {"KAZGUARD" => "true", "HOSTLANG" => ENV['LANG'], "NOKAZ" => ENV['NOKAZ'], "KAZBRANCH" => ENV['KAZBRANCH']} - end -end diff --git a/files/snster-kaz/kaz/prod/provision.sh b/files/snster-kaz/kaz/prod/provision.sh index d1dd130..182fb9c 100644 --- a/files/snster-kaz/kaz/prod/provision.sh +++ b/files/snster-kaz/kaz/prod/provision.sh @@ -18,7 +18,7 @@ DEBIAN_FRONTEND=noninteractive apt-get autoremove -y # KAZ specific things #installation de docker, docker-compose et on y fourre le user debian dans le groupe idoine -DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io docker-compose docker-clean git apg curl sudo unzip rsync btrfs-progs ldap-utils # fuse-overlayfs +DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io docker-compose docker-clean git apg curl sudo unzip rsync btrfs-progs ldap-utils # fuse-overlayfs usermod -G docker debian # activation dans alias dans /root/.bashrc sed -i \ @@ -80,6 +80,27 @@ fi echo -e '#!/bin/sh\n/kaz/bin/container.sh start' >> /etc/rc.local chmod +x /etc/rc.local +# On sauve le proxy APT +proxy=$(/sbin/ip route | awk '/default/ { print $3 }' | head -1) +sed -i -e "s/^proxy.*$/proxy=$proxy/" /usr/local/sbin/detect_proxy.sh +#echo "export http_proxy=\"http://$proxy:3142\"" > /etc/profile.d/proxy.sh +#echo "export https_proxy=\"http://$proxy:3142\"" >> /etc/profile.d/proxy.sh +mkdir /root/.docker +echo "{ + \"proxies\": + { + \"default\": + { + \"httpProxy\": \"http://$proxy:3142\", + \"httpsProxy\": \"http://$proxy:3142\", + \"noProxy\": \"*.sns,127.0.0.0/8\" + } + } +}" > /root/.docker/config.json +echo "http_proxy=\"http://$proxy:3142\" +https_proxy=\"http://$proxy:3142\" +" >> /etc/default/docker + # clear apt cache DEBIAN_FRONTEND=noninteractive apt-get autoremove -y DEBIAN_FRONTEND=noninteractive apt-get clean diff --git a/files/vm-provision.sh b/files/vm-provision.sh index 2b6d5bc..3dc5775 100755 --- a/files/vm-provision.sh +++ b/files/vm-provision.sh @@ -15,52 +15,44 @@ mkdir -p "${VAGRANT_SRC_DIR}/log/" export DebugLog="${VAGRANT_SRC_DIR}/log/log-vagrant-$(date +%y-%m-%d-%T)-" ( echo "########## ********** Start Vagrant $(date +%D-%T)" - #pour la résolution de noms dans /etc/hosts - SERVICES_LIST="smtp mail ldap www depot tableur pad webmail sondage garradin test-garradin wiki git agora cloud office cachet quotas" # Copie de qques fichiers cp "${VAGRANT_SRC_DIR}/keyboard" /etc/default/keyboard - sysctl -w net.ipv4.ip_forward=1 - - # MAJ et install + # gestions sources.list sed -i -e 's/main.*/main contrib non-free/' /etc/apt/sources.list - if [ -f "${VAGRANT_SRC_DIR}/.apt-mirror-config" ]; then - # pour ceux qui disposent d'un cache apt local et pas la fibre - # suffit d'indiquer "host:port" dans le fichier ".apt-mirror-config" - . "${VAGRANT_SRC_DIR}/.apt-mirror-config" - sed -i \ - -e "s%s\?://deb.debian.org%://${APT_MIRROR_DEBIAN}%g" \ - -e "s%s\?://security.debian.org%://${APT_MIRROR_DEBIAN_SECURITY}%g" \ - -e "s%s\?://archive.ubuntu.com%://${APT_MIRROR_UBUNTU}%g" \ - -e "s%s\?://security.ubuntu.com%://${APT_MIRROR_UBUNTU_SECURITY}%g" \ - /etc/apt/sources.list + sed -i -e 's/https:/http:/' /etc/apt/sources.list + apt-get --allow-releaseinfo-change update + + # Cache APT + #DEBIAN_FRONTEND=noninteractive apt-get install -y apt-cacher # apt-cacher-ng does not work well on bullseye + #echo "allowed_hosts = *" >> /etc/apt-cacher/apt-cacher.conf + #service apt-cacher restart + DEBIAN_FRONTEND=noninteractive apt-get install -y squid + sed -i -e "s/#http_access allow localnet/http_access allow localnet/" /etc/squid/squid.conf + echo "cache_dir aufs /var/spool/squid 5000 14 256 +http_port 3142" >> /etc/squid/squid.conf + service squid restart + echo "Acquire::http::Proxy \"http://127.0.0.1:3142\";" > /etc/apt/apt.conf.d/01proxy; # utilisation de apt-cacher-ng + # Ajouter http://www.squid-cache.org/Doc/config/cache_peer/ à squid pour un proxy upstream + + # Pour le confort de chacun, un customVM.sh optionnel + if [ -f "${VAGRANT_SRC_DIR}/customVM.sh" ]; then + bash "${VAGRANT_SRC_DIR}/customVM.sh" fi - DEBIAN_FRONTEND=noninteractive apt-get --allow-releaseinfo-change update + # MAJ et Install DEBIAN_FRONTEND=noninteractive apt-get -y upgrade DEBIAN_FRONTEND=noninteractive apt-get -y dist-upgrade - DEBIAN_FRONTEND=noninteractive apt-get install -y apg curl git sudo unzip rsync firefox-esr tcpdump net-tools mousepad wireshark swapspace whois ldap-utils python3-lxc lxc python3-pygraphviz python3-pil python3-yaml imagemagick btrfs-progs # could be with --no-install-recommends + DEBIAN_FRONTEND=noninteractive apt-get install -y apg curl git sudo unzip rsync firefox-esr tcpdump net-tools mousepad wireshark swapspace whois python3-lxc lxc python3-pygraphviz python3-pil python3-yaml imagemagick btrfs-progs # could be with --no-install-recommends DEBIAN_FRONTEND=noninteractive apt-get install -y xfce4 lightdm xfce4-terminal xserver-xorg gitk # needs to install recommends ssh-keygen -t rsa -b 4096 -N '' <<<$'\ny' rsync /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys - # Pour le confort de chacun - # Le fihcier .customDocker.sh contient - # DEBIAN_FRONTEND=noninteractive apt-get install -y joe - # DEBIAN_FRONTEND=noninteractive apt-get install -y emacs - # DEBIAN_FRONTEND=noninteractive apt-get install -y vim - if [ -f "${VAGRANT_SRC_DIR}/.customDocker.sh" ]; then - chmod a+x "${VAGRANT_SRC_DIR}/.customDocker.sh" - "${VAGRANT_SRC_DIR}/.customDocker.sh" - fi - - - # Localisation du $LANG, en par défaut, timezone Paris if [ -z "${HOSTLANG}" ] ; then - HOSTLANG="en_US.UTF-8" + HOSTLANG="en_US.UTF-8" fi echo "Europe/Paris" > /etc/timezone ln -sf /usr/share/zoneinfo/Europe/Paris /etc/localtime @@ -71,38 +63,33 @@ export DebugLog="${VAGRANT_SRC_DIR}/log/log-vagrant-$(date +%y-%m-%d-%T)-" dpkg-reconfigure --frontend=noninteractive locales || true # don't fail for a locales problem update-locale LANG=${HOSTLANG} || true # don't fail for a locales problem - - echo -e "\n #### create user\n" # Creation des utilisateurs + echo -e "\n #### create user\n" usermod -p $(mkpasswd --method=sha-512 root) root useradd -m -s "/bin/bash" -p $(mkpasswd --method=sha-512 debian) debian || true # don't fail if user already exists - # augmentation de la taille de /run si lowmem - #echo "tmpfs /run tmpfs nosuid,noexec,size=26M 0 0" >> /etc/fstab - #mount -o remount /run - # Désactivation de la mise en veille de l'écran mkdir -p /etc/X11/xorg.conf.d/ rsync -a "${VAGRANT_SRC_DIR}/10-monitor.conf" /etc/X11/xorg.conf.d/ # mv /etc/xdg/autostart/light-locker.desktop /etc/xdg/autostart/light-locker.desktop.bak DEBIAN_FRONTEND=noninteractive apt-get remove --purge -y light-locker - #faut virer exim, il fout la grouille avec le docker postfix + #faut virer exim, inutile DEBIAN_FRONTEND=noninteractive apt-get remove --purge -y exim4-base exim4-config exim4-daemon-light #login ssh avec mot de passe sed -i "s/PasswordAuthentication no/PasswordAuthentication yes/" /etc/ssh/sshd_config if ! grep -q "PasswordAuthentication yes" /etc/ssh/sshd_config 2>/dev/null; then - echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config + echo "PasswordAuthentication yes" >> /etc/ssh/sshd_config fi # autorisation du routing et augmentation inotify if ! grep -q "net.ipv4.ip_forward" /etc/sysctl.conf 2>/dev/null; then - echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf + echo "net.ipv4.ip_forward=1" >> /etc/sysctl.conf fi sed -i "s/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/" /etc/sysctl.conf if ! grep -q "fs.inotify.max_queued_events" /etc/sysctl.conf 2>/dev/null; then - echo -e "fs.inotify.max_queued_events=1048576\nfs.inotify.max_user_instances=1048576\nfs.inotify.max_user_watches=1048576" >> /etc/sysctl.conf + echo -e "fs.inotify.max_queued_events=1048576\nfs.inotify.max_user_instances=1048576\nfs.inotify.max_user_watches=1048576" >> /etc/sysctl.conf fi sysctl -p @@ -128,50 +115,45 @@ EOF mkdir -p $(dirname "${TERM_CFG}") touch "${TERM_CFG}" if ! grep -q "ShortcutsNoMnemonics" "${TERM_CFG}" 2>/dev/null; then - echo -e "[Configuration]\nShortcutsNoMnemonics=TRUE" >> "${TERM_CFG}" + echo -e "[Configuration]\nShortcutsNoMnemonics=TRUE" >> "${TERM_CFG}" fi - echo -e "\n #### set swapspace\n" # free swapspace at shutdown + echo -e "\n #### set swapspace\n" sed -i -e 's/ExecStart=\/usr\/sbin\/swapspace/ExecStart=\/usr\/sbin\/swapspace\nExecStop=\/usr\/sbin\/swapspace -e/' /lib/systemd/system/swapspace.service systemctl daemon-reload # limit journald log size mkdir -p /etc/systemd/journald.conf.d if [ ! -f /etc/systemd/journald.conf.d/sizelimit.conf ]; then - cat > /etc/systemd/journald.conf.d/sizelimit.conf < /etc/systemd/journald.conf.d/sizelimit.conf <> /etc/fstab mount /var/lib/lxc - #losetup -f /root/btrfs.img - #mount /dev/loop0 /var/lib/lxc sed -i -e "s/template=self.template/template=self.template, bdevtype='btrfs'/" /usr/local/lib/python3.9/dist-packages/backends/LxcBackend.py # SNSTER KAZ - # cp -ar ${VAGRANT_SRC_DIR}/templates /root cp -ar ${VAGRANT_SRC_DIR}/snster-kaz /root # crypto keys cp -ar /etc/letsencrypt /root/snster-kaz/kaz/prod/ cp -ar /etc/letsencrypt /root/snster-kaz/isp-a/home/ - # On monte le filesystem de kaz-prod dans le /kaz de la VM pour le dév (en nofail) -# mkdir /kaz-prod /kaz -# echo "overlay /kaz-prod overlay lowerdir=/var/lib/lxc/sr-masters-bullseye/rootfs,upperdir=/var/lib/lxc/kaz-kaz-prod/overlay/delta,workdir=/var/lib/lxc/kaz-kaz-prod/overlay/work,nofail 0 0" >> /etc/fstab -# echo "/kaz-prod/kaz /kaz none bind,nofail 0 0" >> /etc/fstab + # On lie le filesystem de kaz-prod dans le /kaz de la VM pour le dév ln -s /var/lib/lxc/kaz-kaz-prod/rootfs/ /kaz-prod ln -s /kaz-prod/kaz /kaz diff --git a/init.sh b/init.sh deleted file mode 100755 index 285a2b0..0000000 --- a/init.sh +++ /dev/null @@ -1,162 +0,0 @@ -#!/bin/bash - -### Personalisation de la VM - -cd "$(dirname $0)" - -BOLD='' -RED='' -GREEN='' -YELLOW='' -BLUE='' -MAGENTA='' -CYAN='' -NC='' # No Color -NL=' -' - -mkdir -p ./files/kaz/log/ ./files/kaz/download/ ./files/kaz/git/ -chmod a+rxw ./files/kaz/log/ ./files/kaz/download/ ./files/kaz/git/ - -cp Vagrantfile.dist Vagrantfile - -OLD_MEN=$(grep vb.memory Vagrantfile | sed -e 's%.*vb.memory\s*=\s*"\([^"]*\)".*%\1%') -OLD_CUPS=$(grep vb.cpus Vagrantfile | sed -e 's%.*vb.cpus\s*=\s*"\([^"]*\)".*%\1%') -MEM=$(expr $(head -1 /proc/meminfo | awk '{print $2}') / 4096) -CUP=$(expr $(nproc) / 2) - -cat < ${MEM} - CUP: ${OLD_CUPS} => ${CUP} -${NC} -Update './Vagrantfile' -EOF - -sed -i Vagrantfile \ - -e 's%vb.memory\s*=\s*"[^"]*"%vb.memory = "'${MEM}'"%' \ - -e 's%vb.cpus\s*=\s*"[^"]*"%vb.cpus = "'${CUP}'"%' - - -APT_CONF="files/.apt-mirror-config" -if [ -f "${APT_CONF}" ]; then - . "${APT_CONF}" -fi - -### Personalisation d'un cache apt -if [ -z "${APT_MIRROR_DEBIAN}" ]; then - APT_MIRROR_DEBIAN=$(grep "deb\s.*/debian[^-]" /etc/apt/sources.list | head -1 | sed -e "s%.*deb\s.*://\([^/]*\)/debian.*%\1%") -fi -if [ -z "${APT_MIRROR_DEBIAN_SECURITY}" ]; then - APT_MIRROR_DEBIAN_SECURITY=$(grep "deb\s.*/debian-security" /etc/apt/sources.list | head -1 | sed -e "s%.*deb\s.*://\([^/]*\)/debian-security.*%\1%") -fi -if [ -z "${APT_MIRROR_UBUNTU}" ]; then - APT_MIRROR_UBUNTU=$(grep "deb\s.*://\([^/]*\)/ubuntu" /etc/apt/sources.list | head -1 | sed -e "s%.*deb\s.*://\([^/]*\)/ubuntu.*%\1%") -fi -if [ -z "${APT_MIRROR_UBUNTU_SECURITY}" ]; then - APT_MIRROR_UBUNTU_SECURITY=$(grep "deb\s.*://\([^/]*\)/ubuntu.*-security" /etc/apt/sources.list | head -1 | sed -e "s%.*deb\s.*://\([^/]*\)/ubuntu.*%\1%") -fi - -if [ -z "${APT_MIRROR_UBUNTU}" ]; then - APT_MIRROR_UBUNTU="${APT_MIRROR_DEBIAN}" -fi -if [ -z "${APT_MIRROR_UBUNTU_SECURITY}" ]; then - APT_MIRROR_UBUNTU_SECURITY="${APT_MIRROR_DEBIAN_SECURITY}" -fi - -while : ; do - cat < "${APT_CONF}" < "${PROXY_CONF}" < "${CUSTOM_CONF}" - fi - chmod a+x "${CUSTOM_CONF}" - if ! grep -qw "${editor}" "${CUSTOM_CONF}" 2> /dev/null ; then - echo "DEBIAN_FRONTEND=noninteractive apt-get install -y ${editor}" >> "${CUSTOM_CONF}" - echo "rsync -a /vagrant/files/.emacs* /root/" >> "${CUSTOM_CONF}" - fi - break;; - ""|[Nn]* ) break;; - * ) echo "Please answer joe, emacs, vim or no.";; - esac -done