From d3199dbf0929ef7111ed57014fc98ab5853d2173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois?= Date: Thu, 10 Feb 2022 01:40:17 +0100 Subject: [PATCH] depollueur v2 --- bin/installDepollueur.sh | 25 ++++++++ dockers/jirafeau/build.sh | 15 ++--- dockers/jirafeau/config/jirafeau.conf | 4 +- dockers/postfix/Dockerfile | 3 +- dockers/postfix/build.sh | 5 ++ dockers/postfix/docker-compose.yml | 9 ++- dockers/postfix/filter/domainname | 1 + dockers/postfix/filter/filter.sh | 84 +++++++++++++++++---------- dockers/sympa/build.sh | 5 ++ dockers/sympa/docker-compose.yml | 8 ++- secret.tmpl/SetAllPass.sh | 2 +- secret.tmpl/env-roundcubeServ | 1 + 12 files changed, 112 insertions(+), 50 deletions(-) create mode 100755 bin/installDepollueur.sh create mode 100644 dockers/postfix/filter/domainname mode change 100755 => 100644 dockers/postfix/filter/filter.sh diff --git a/bin/installDepollueur.sh b/bin/installDepollueur.sh new file mode 100755 index 0000000..80247f7 --- /dev/null +++ b/bin/installDepollueur.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +KAZ_ROOT=$(cd "$(dirname $0)/.."; pwd) +. "${KAZ_ROOT}/bin/.commonFunctions.sh" +setKazVars + +if [[ -x "${KAZ_GIT_DIR}/depollueur/build/out/eMailShrinker" ]]; then + exit +fi + +printKazMsg "\n *** Installation du dépollueur" + +sudo apt-get install -y --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev + +mkdir -p "${KAZ_GIT_DIR}" +cd "${KAZ_GIT_DIR}" +if [ ! -d "depollueur" ]; then + git clone "${SRC_DEP}" +fi +cd depollueur +git reset --hard && git pull +make + +. "${DOCKERS_ENV}" +echo "${domain}" > "src/bash/domainname" diff --git a/dockers/jirafeau/build.sh b/dockers/jirafeau/build.sh index 8b1a4bb..91fb5f5 100755 --- a/dockers/jirafeau/build.sh +++ b/dockers/jirafeau/build.sh @@ -9,16 +9,9 @@ SRC_JIR="https://gitlab.com/mojo42/Jirafeau.git" SRC_DEP="https://git.kaz.bzh/KAZ/depollueur.git" JIR_VER="4.3.0" -printKazMsg "\n *** Création du Dockerfile Jirafeau" +"${KAZ_BIN_DIR}/installDepollueur.sh" -printKazMsg "\n - GIT dépollueur " -mkdir -p "${KAZ_GIT_DIR}" -cd "${KAZ_GIT_DIR}" -if [ ! -d "depollueur" ]; then - git clone "${SRC_DEP}" -fi -cd depollueur -git reset --hard && git pull +printKazMsg "\n *** Création du Dockerfile Jirafeau" printKazMsg "\n - GIT Jirafeau " cd "${KAZ_GIT_DIR}" @@ -27,9 +20,9 @@ if [ ! -d "Jirafeau" ]; then fi cd "${KAZ_GIT_DIR}/Jirafeau" && git reset --hard && git checkout ${JIR_VER} -cd "${KAZ_GIT_DIR}/Jirafeau" && rsync -a ../depollueur/src/Jirafeau/[ft].php . +cd "${KAZ_GIT_DIR}/Jirafeau" && rsync -a ../depollueur/src/Jirafeau/[aft].php . -printKazMsg "\n - Dockefile " +printKazMsg "\n - Dockefile" cd "${KAZ_ROOT}" # Pour permettre la copy de git il faut que le répertoire soit visible de la racine qui lance la construction docker build -t filekaz . -f dockers/jirafeau/Dockerfile diff --git a/dockers/jirafeau/config/jirafeau.conf b/dockers/jirafeau/config/jirafeau.conf index 1d26c7d..0821467 100644 --- a/dockers/jirafeau/config/jirafeau.conf +++ b/dockers/jirafeau/config/jirafeau.conf @@ -1,8 +1,8 @@ ServerName file.kaz.bzh DocumentRoot /var/jirafeau/ - ErrorLog ${APACHE_LOG_DIR}/date-error.log - CustomLog ${APACHE_LOG_DIR}/date-access.log combined + ErrorLog ${APACHE_LOG_DIR}/error.log + CustomLog ${APACHE_LOG_DIR}/access.log combined ServerSignature Off diff --git a/dockers/postfix/Dockerfile b/dockers/postfix/Dockerfile index da6183e..a9d455d 100644 --- a/dockers/postfix/Dockerfile +++ b/dockers/postfix/Dockerfile @@ -24,7 +24,8 @@ RUN update-locale LANG=fr_FR.UTF-8 RUN apt-get -y install rsyslog apt-utils apg gawk altermime RUN apt-get -y install libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev -RUN apt-get -y install emacs elpa-php-mode vim nano mailutils bsd-mailx vim procps +#RUN apt-get -y install emacs elpa-php-mode +RUN apt-get -y install vim nano mailutils bsd-mailx procps # creation du user filter,son repertoire home, copie des fichiers RUN mkdir /home/filter ; useradd -d /home/filter filter ; chown filter /home/filter diff --git a/dockers/postfix/build.sh b/dockers/postfix/build.sh index ccf6418..7a8a4d1 100755 --- a/dockers/postfix/build.sh +++ b/dockers/postfix/build.sh @@ -4,8 +4,13 @@ KAZ_ROOT=$(cd $(dirname $0)/../..; pwd) . "${KAZ_ROOT}/bin/.commonFunctions.sh" setKazVars +"${KAZ_BIN_DIR}/installDepollueur.sh" + printKazMsg "\n *** Création du Dockerfile Postfix" cd "${KAZ_ROOT}" +mkdir -p dockers/postfix/filter/ +rsync -a git/depollueur/src/bash/* git/depollueur/build/out/* dockers/postfix/filter/ + docker build -t postfixkaz . -f dockers/postfix/Dockerfile diff --git a/dockers/postfix/docker-compose.yml b/dockers/postfix/docker-compose.yml index 382a716..6a77bad 100644 --- a/dockers/postfix/docker-compose.yml +++ b/dockers/postfix/docker-compose.yml @@ -2,13 +2,15 @@ version: '3.3' services: mail: - #image: tvial/docker-mailserver:latest image: postfixkaz hostname: ${smtpHost} domainname: ${domain} container_name: ${smtpServName} networks: - postfixNet + - jirafeauNet + external_links: + - ${jirafeauServName}:${fileHost} ports: - ${MAIN_IP}:25:25 - ${MAIN_IP}:143:143 @@ -20,10 +22,8 @@ services: - mailLog:/var/log/mail - mailEtc:/etc - mailConfig:/tmp/docker-mailserver/ - #- ./config/:/tmp/docker-mailserver/ - /etc/localtime:/etc/localtime:ro - /etc/timezone:/etc/timezone:ro -# - /etc/ssl:/tmp/ssl:ro - /etc/letsencrypt:/etc/letsencrypt:ro environment: - HOSTNAME=${smtpHost} @@ -47,4 +47,7 @@ networks: postfixNet: external: name: postfixNet + jirafeauNet: + external: + name: jirafeauNet diff --git a/dockers/postfix/filter/domainname b/dockers/postfix/filter/domainname new file mode 100644 index 0000000..77d056a --- /dev/null +++ b/dockers/postfix/filter/domainname @@ -0,0 +1 @@ +kaz.local diff --git a/dockers/postfix/filter/filter.sh b/dockers/postfix/filter/filter.sh old mode 100755 new mode 100644 index f4ad985..5997432 --- a/dockers/postfix/filter/filter.sh +++ b/dockers/postfix/filter/filter.sh @@ -42,8 +42,8 @@ ########################################################################## cd $(dirname $0) - -# Exit coINSPECT_DIRdes from +DOMAINNAME=$(cat domainname) +# Exit codes from EX_TEMPFAIL=75 EX_UNAVAILABLE=69 EX_TOO_LARGE=552 @@ -55,8 +55,8 @@ MAX_KEEP_IN_MAIL=5ki MAX_UPLOAD_SIZE=100Mi SHRINK_CMD=/home/filter/eMailShrinker JIRAFEAU_CMD=/home/filter/jirafeauAPI -JIRAFEAU_OLDURL="https://\(file\|depot\)\.kaz\.bzh" -JIRAFEAU_URL=https://depot.kaz.bzh +JIRAFEAU_URL=https://depot.${DOMAINNAME:-"kaz.bzh"} +JIRAFEAU_LOCAL=http://depot JIRAFEAU_TIME=month MD5_CMD=/usr/bin/md5sum DISCLAMER_CMD=altermime @@ -78,10 +78,12 @@ DATE_TEMPS=$(date "+%Y-%m-%d-%H:%M:%S") REP_PIECE_JOINTE=$(echo "${MAILS}/${DATE_TEMPS}_${MAIL_SOURCE}_$$") cd "${INSPECT_DIR}" || { echo "${INSPECT_DIR} does not exist"; exit "${EX_TEMPFAIL}"; } +# lien renvoyé par le téléverssement ONE_LINK="${REP_PIECE_JOINTE}/one.txt" -ALL_LINKS="${REP_PIECE_JOINTE}/url-list.txt" +# anciens liens à réactiver OLD_LINKS="${REP_PIECE_JOINTE}/url-to-refresh.txt" -PREV_CODES="${REP_PIECE_JOINTE}/prev-codes.txt" +# contenu de l'archive +ARCHIVE_CONTENT="${REP_PIECE_JOINTE}/archive-content.txt" # Clean up when done or when aborting. trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15 @@ -91,27 +93,28 @@ cat > "in.$$" || { LOG_FIC "Cannot save mail to file"; exit "${EX_TEMPFAIL}"; } # cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.bak" mkdir -p "${REP_PIECE_JOINTE}/" ->"${ALL_LINKS}" >"${OLD_LINKS}" ->"${PREV_CODES}" +>"${ARCHIVE_CONTENT}" # Etape de rafraichissement des anciens fichiers inclus -OLD_CODES="" +cat > "${ARCHIVE_CONTENT}" <> \"${FIC_LOG}\" > \"${OLD_LINKS}\"" "${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" > "${OLD_LINKS}" -cat "${OLD_LINKS}" | grep "${JIRAFEAU_OLDURL}" | while read REMOTE_LINK +cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK do REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//') [ -z "${REMOTE_REF}" ] && continue REMOTE_KEY=$(echo "${REMOTE_LINK}" | grep "k=" | sed 's%.*k=\([^&]*\).*%\1%') # update periode for download - LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_URL}\" -t \"${JIRAFEAU_TIME}\" update \"${REMOTE_REF}\" 2>&1 >> \"${FIC_LOG}\"" - "${JIRAFEAU_CMD}" -f "${JIRAFEAU_URL}" -t "${JIRAFEAU_TIME}" update "${REMOTE_REF}" 2>&1 >> "${FIC_LOG}" - echo -n "/${REMOTE_REF}~${REMOTE_KEY}" >> "${PREV_CODES}" + LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -t \"${JIRAFEAU_TIME}\" update \"${REMOTE_REF}\" 2>&1 >> \"${FIC_LOG}\"" + "${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -t "${JIRAFEAU_TIME}" update "${REMOTE_REF}" 2>&1 >> "${FIC_LOG}" + echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "${ARCHIVE_CONTENT}" done -OLD_CODES=$(cat "${PREV_CODES}") -LOG_FIC " - OLD_CODES=${OLD_CODES}" +LOG_FIC " - archive starts with: $(cat ${ARCHIVE_CONTENT})" # Etape extraction des pieces jointes LOG_FIC "${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_DIR}/in.$$" @@ -124,46 +127,67 @@ LOG_FIC "${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_D ATTACH_NAME=$(grep "^Name: " "${ATTACH_TMP_NAME}/meta" | cut -c 7- ) ATTACH_CONTENT_TYPE=$(grep "^Content-Type: " "${ATTACH_TMP_NAME}/meta" | cut -c 15- ) else - # XXX a virer - ATTACH_MEDIA="${ATTACH_TMP_NAME}" - ATTACH_NAME=$(basename "${ATTACH_MEDIA}") - ATTACH_CONTENT_TYPE="" + # XXX error + continue fi # Etape de televersement des pieces jointes PASSWORD=$(apg -n 1 -m 12) PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1) - actualSize=$(ls -l "${ATTACH_MEDIA}") - LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_URL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ATTACH_CONTENT_TYPE}\" -n \"${ATTACH_NAME}\" send \"${ATTACH_MEDIA}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\"" - "${JIRAFEAU_CMD}" -f "${JIRAFEAU_URL}" -s "${MAX_UPLOAD_SIZE}" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}" + LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ATTACH_CONTENT_TYPE}\" -n \"${ATTACH_NAME}\" send \"${ATTACH_MEDIA}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\"" + "${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -s "${MAX_UPLOAD_SIZE}" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}" cat "${ONE_LINK}" | { read JIR_TOKEN read JIR_CODE - LOG_FIC " - Jirafeau envoie ${JIR_TOKEN} et ${JIR_CODE}" + LOG_FIC " - Jirafeau retourne ${JIR_TOKEN} et ${JIR_CODE}" case "${JIR_TOKEN}" in "" | no | *Error* | \<* ) LOG_FIC " - impossible de televerser ${ATTACH_TMP_FILE} (${JIR_TOKEN}), il ne sera pas remplace dans le message" - echo "" + echo "new:" ;; * ) LOG_FIC " - substitution par ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}" - echo "${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5} /${JIR_TOKEN}~${PASSWORD_MD5}" - echo "${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}" >> "${ALL_LINKS}" + echo "url: ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}" + echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "${ARCHIVE_CONTENT}" ;; esac } LOG_FIC " - supprimer l'extraction ${ATTACH_TMP_FILE}" rm -f "${ATTACH_TMP_FILE}" done + # Création de l'archive + if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 4 ]; then + PASSWORD=$(apg -n 1 -m 12) + PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1) + LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"text/plain\" -n \"archive_content\" send \"${ARCHIVE_CONTENT}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\"" + "${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -s "${MAX_UPLOAD_SIZE}" -c "text/plain" -n "archive_content" send "${ARCHIVE_CONTENT}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}" + cat "${ONE_LINK}" | { + read JIR_TOKEN + read JIR_CODE + LOG_FIC " - Jirafeau retourne ${JIR_TOKEN} et ${JIR_CODE}" + case "${JIR_TOKEN}" in + "" | no | *Error* | \<* ) + LOG_FIC " - impossible de televerser l'archive (${JIR_TOKEN}), il ne sera pas remplace dans le message" + echo "arch: bad" + ;; + * ) + LOG_FIC " - ajoute de l'archive ${JIRAFEAU_URL}/a.php?g=${JIR_TOKEN}~${PASSWORD_MD5}" + echo "arch: ${JIRAFEAU_URL}/a.php?g=${JIR_TOKEN}~${PASSWORD_MD5}" + ;; + esac + } + else + LOG_FIC " - pas d'archive (moins de 2 PJ)" + echo "arch: none" + fi # Etape de substitution - LOG_FIC "${SHRINK_CMD} -a \"${JIRAFEAU_URL}/t.php?n=${MAIL_SOURCE}_${DATE_TEMPS}&l=${OLD_CODES}\" -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"" -} | "${SHRINK_CMD}" -a "${JIRAFEAU_URL}/t.php?n=${MAIL_SOURCE}_${DATE_TEMPS}&l=${OLD_CODES}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}" + LOG_FIC "${SHRINK_CMD} -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"" +} | "${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}" # XXX trace # cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" /var/mail/tmp/ # Etape choix de modification du message d'origine -if [ -s "${ALL_LINKS}" -o -s "${OLD_LINKS}" ] -then +if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 3 ]; then # verification de taille finale actualSize=$(wc -c < "${INSPECT_DIR}/in.$$.altered") if [ ${actualSize} -ge $MAX_FINAL_SIZE ]; then diff --git a/dockers/sympa/build.sh b/dockers/sympa/build.sh index 444bd65..6f7d911 100755 --- a/dockers/sympa/build.sh +++ b/dockers/sympa/build.sh @@ -4,8 +4,13 @@ KAZ_ROOT=$(cd $(dirname $0)/../..; pwd) . "${KAZ_ROOT}/bin/.commonFunctions.sh" setKazVars +"${KAZ_BIN_DIR}/installDepollueur.sh" + printKazMsg "\n *** Création du Dockerfile Sympa" cd "${KAZ_ROOT}" +mkdir -p dockers/postfix/filter/ +rsync -a git/depollueur/src/bash/* git/depollueur/build/out/* dockers/sympa/filter/ + docker build -t sympakaz . -f dockers/sympa/Dockerfile diff --git a/dockers/sympa/docker-compose.yml b/dockers/sympa/docker-compose.yml index 4da7a28..3670fd5 100644 --- a/dockers/sympa/docker-compose.yml +++ b/dockers/sympa/docker-compose.yml @@ -1,6 +1,7 @@ version: '3.3' services: + # la DB est-elle utile ? db: image: mariadb:10.5 container_name: ${sympaDBName} @@ -21,6 +22,9 @@ services: restart: ${restartPolicy} networks: - sympaNet + - jirafeauNet + external_links: + - ${jirafeauServName}:${fileHost} ports: - ${SYMPA_IP}:25:25 #- ${SYMPA_IP}:143:143 @@ -47,9 +51,9 @@ services: - /etc/timezone:/etc/timezone:ro #- /etc/ssl:/etc/ssl:ro - /etc/letsencrypt:/etc/letsencrypt:ro - # environment: + environment: + - DOMAINNAME=${doamin} # - HOSTNAME=${sympaHost} - # - DOMAINNAME=${sympaDomain} # - CONTAINER_NAME=${sympaServName} # env_file: # - ../../secret/env-${sympaServName} diff --git a/secret.tmpl/SetAllPass.sh b/secret.tmpl/SetAllPass.sh index d84de21..165dfa2 100644 --- a/secret.tmpl/SetAllPass.sh +++ b/secret.tmpl/SetAllPass.sh @@ -152,12 +152,12 @@ roundcube_ROUNDCUBEMAIL_DB_TYPE="mysql" roundcube_ROUNDCUBEMAIL_DB_NAME="${roundcube_MYSQL_DATABASE}" roundcube_ROUNDCUBEMAIL_DB_USER="${roundcube_MYSQL_USER}" roundcube_ROUNDCUBEMAIL_DB_PASSWORD="${roundcube_MYSQL_PASSWORD}" +roundcube_ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE="100Mo" # XXX TODO >>> # ROUNDCUBEMAIL_DB_PORT # ROUNDCUBEMAIL_PLUGINS -# ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE # ROUNDCUBEMAIL_SPELLCHECK_URI # ROUNDCUBEMAIL_ASPELL_DICTS # XXX TODO <<< diff --git a/secret.tmpl/env-roundcubeServ b/secret.tmpl/env-roundcubeServ index 50c4d03..3f3cf66 100644 --- a/secret.tmpl/env-roundcubeServ +++ b/secret.tmpl/env-roundcubeServ @@ -3,3 +3,4 @@ ROUNDCUBEMAIL_DB_TYPE= ROUNDCUBEMAIL_DB_NAME= ROUNDCUBEMAIL_DB_USER= ROUNDCUBEMAIL_DB_PASSWORD= +ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE=