Browse Source

depollueur v2

pull/3/head
François 2 years ago
parent
commit
d3199dbf09
  1. 25
      bin/installDepollueur.sh
  2. 15
      dockers/jirafeau/build.sh
  3. 4
      dockers/jirafeau/config/jirafeau.conf
  4. 3
      dockers/postfix/Dockerfile
  5. 5
      dockers/postfix/build.sh
  6. 9
      dockers/postfix/docker-compose.yml
  7. 1
      dockers/postfix/filter/domainname
  8. 84
      dockers/postfix/filter/filter.sh
  9. 5
      dockers/sympa/build.sh
  10. 8
      dockers/sympa/docker-compose.yml
  11. 2
      secret.tmpl/SetAllPass.sh
  12. 1
      secret.tmpl/env-roundcubeServ

25
bin/installDepollueur.sh

@ -0,0 +1,25 @@
#!/bin/bash
KAZ_ROOT=$(cd "$(dirname $0)/.."; pwd)
. "${KAZ_ROOT}/bin/.commonFunctions.sh"
setKazVars
if [[ -x "${KAZ_GIT_DIR}/depollueur/build/out/eMailShrinker" ]]; then
exit
fi
printKazMsg "\n *** Installation du dépollueur"
sudo apt-get install -y --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev
mkdir -p "${KAZ_GIT_DIR}"
cd "${KAZ_GIT_DIR}"
if [ ! -d "depollueur" ]; then
git clone "${SRC_DEP}"
fi
cd depollueur
git reset --hard && git pull
make
. "${DOCKERS_ENV}"
echo "${domain}" > "src/bash/domainname"

15
dockers/jirafeau/build.sh

@ -9,16 +9,9 @@ SRC_JIR="https://gitlab.com/mojo42/Jirafeau.git"
SRC_DEP="https://git.kaz.bzh/KAZ/depollueur.git"
JIR_VER="4.3.0"
printKazMsg "\n *** Création du Dockerfile Jirafeau"
"${KAZ_BIN_DIR}/installDepollueur.sh"
printKazMsg "\n - GIT dépollueur "
mkdir -p "${KAZ_GIT_DIR}"
cd "${KAZ_GIT_DIR}"
if [ ! -d "depollueur" ]; then
git clone "${SRC_DEP}"
fi
cd depollueur
git reset --hard && git pull
printKazMsg "\n *** Création du Dockerfile Jirafeau"
printKazMsg "\n - GIT Jirafeau "
cd "${KAZ_GIT_DIR}"
@ -27,9 +20,9 @@ if [ ! -d "Jirafeau" ]; then
fi
cd "${KAZ_GIT_DIR}/Jirafeau" && git reset --hard && git checkout ${JIR_VER}
cd "${KAZ_GIT_DIR}/Jirafeau" && rsync -a ../depollueur/src/Jirafeau/[ft].php .
cd "${KAZ_GIT_DIR}/Jirafeau" && rsync -a ../depollueur/src/Jirafeau/[aft].php .
printKazMsg "\n - Dockefile "
printKazMsg "\n - Dockefile"
cd "${KAZ_ROOT}"
# Pour permettre la copy de git il faut que le répertoire soit visible de la racine qui lance la construction
docker build -t filekaz . -f dockers/jirafeau/Dockerfile

4
dockers/jirafeau/config/jirafeau.conf

@ -1,8 +1,8 @@
<VirtualHost *:80>
ServerName file.kaz.bzh
DocumentRoot /var/jirafeau/
ErrorLog ${APACHE_LOG_DIR}/date-error.log
CustomLog ${APACHE_LOG_DIR}/date-access.log combined
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog ${APACHE_LOG_DIR}/access.log combined
ServerSignature Off
<Location / >

3
dockers/postfix/Dockerfile

@ -24,7 +24,8 @@ RUN update-locale LANG=fr_FR.UTF-8
RUN apt-get -y install rsyslog apt-utils apg gawk altermime
RUN apt-get -y install libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev
RUN apt-get -y install emacs elpa-php-mode vim nano mailutils bsd-mailx vim procps
#RUN apt-get -y install emacs elpa-php-mode
RUN apt-get -y install vim nano mailutils bsd-mailx procps
# creation du user filter,son repertoire home, copie des fichiers
RUN mkdir /home/filter ; useradd -d /home/filter filter ; chown filter /home/filter

5
dockers/postfix/build.sh

@ -4,8 +4,13 @@ KAZ_ROOT=$(cd $(dirname $0)/../..; pwd)
. "${KAZ_ROOT}/bin/.commonFunctions.sh"
setKazVars
"${KAZ_BIN_DIR}/installDepollueur.sh"
printKazMsg "\n *** Création du Dockerfile Postfix"
cd "${KAZ_ROOT}"
mkdir -p dockers/postfix/filter/
rsync -a git/depollueur/src/bash/* git/depollueur/build/out/* dockers/postfix/filter/
docker build -t postfixkaz . -f dockers/postfix/Dockerfile

9
dockers/postfix/docker-compose.yml

@ -2,13 +2,15 @@ version: '3.3'
services:
mail:
#image: tvial/docker-mailserver:latest
image: postfixkaz
hostname: ${smtpHost}
domainname: ${domain}
container_name: ${smtpServName}
networks:
- postfixNet
- jirafeauNet
external_links:
- ${jirafeauServName}:${fileHost}
ports:
- ${MAIN_IP}:25:25
- ${MAIN_IP}:143:143
@ -20,10 +22,8 @@ services:
- mailLog:/var/log/mail
- mailEtc:/etc
- mailConfig:/tmp/docker-mailserver/
#- ./config/:/tmp/docker-mailserver/
- /etc/localtime:/etc/localtime:ro
- /etc/timezone:/etc/timezone:ro
# - /etc/ssl:/tmp/ssl:ro
- /etc/letsencrypt:/etc/letsencrypt:ro
environment:
- HOSTNAME=${smtpHost}
@ -47,4 +47,7 @@ networks:
postfixNet:
external:
name: postfixNet
jirafeauNet:
external:
name: jirafeauNet

1
dockers/postfix/filter/domainname

@ -0,0 +1 @@
kaz.local

84
dockers/postfix/filter/filter.sh

@ -42,8 +42,8 @@
##########################################################################
cd $(dirname $0)
# Exit coINSPECT_DIRdes from <sysexits.h>
DOMAINNAME=$(cat domainname)
# Exit codes from <sysexits.h>
EX_TEMPFAIL=75
EX_UNAVAILABLE=69
EX_TOO_LARGE=552
@ -55,8 +55,8 @@ MAX_KEEP_IN_MAIL=5ki
MAX_UPLOAD_SIZE=100Mi
SHRINK_CMD=/home/filter/eMailShrinker
JIRAFEAU_CMD=/home/filter/jirafeauAPI
JIRAFEAU_OLDURL="https://\(file\|depot\)\.kaz\.bzh"
JIRAFEAU_URL=https://depot.kaz.bzh
JIRAFEAU_URL=https://depot.${DOMAINNAME:-"kaz.bzh"}
JIRAFEAU_LOCAL=http://depot
JIRAFEAU_TIME=month
MD5_CMD=/usr/bin/md5sum
DISCLAMER_CMD=altermime
@ -78,10 +78,12 @@ DATE_TEMPS=$(date "+%Y-%m-%d-%H:%M:%S")
REP_PIECE_JOINTE=$(echo "${MAILS}/${DATE_TEMPS}_${MAIL_SOURCE}_$$")
cd "${INSPECT_DIR}" || { echo "${INSPECT_DIR} does not exist"; exit "${EX_TEMPFAIL}"; }
# lien renvoyé par le téléverssement
ONE_LINK="${REP_PIECE_JOINTE}/one.txt"
ALL_LINKS="${REP_PIECE_JOINTE}/url-list.txt"
# anciens liens à réactiver
OLD_LINKS="${REP_PIECE_JOINTE}/url-to-refresh.txt"
PREV_CODES="${REP_PIECE_JOINTE}/prev-codes.txt"
# contenu de l'archive
ARCHIVE_CONTENT="${REP_PIECE_JOINTE}/archive-content.txt"
# Clean up when done or when aborting.
trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
@ -91,27 +93,28 @@ cat > "in.$$" || { LOG_FIC "Cannot save mail to file"; exit "${EX_TEMPFAIL}"; }
# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.bak"
mkdir -p "${REP_PIECE_JOINTE}/"
>"${ALL_LINKS}"
>"${OLD_LINKS}"
>"${PREV_CODES}"
>"${ARCHIVE_CONTENT}"
# Etape de rafraichissement des anciens fichiers inclus
OLD_CODES=""
cat > "${ARCHIVE_CONTENT}" <<EOF
src: ${MAIL_SOURCE}
time: ${DATE_TEMPS}
EOF
LOG_FIC "${SHRINK_CMD} -u \"${INSPECT_DIR}/in.$$\" 2>> \"${FIC_LOG}\" > \"${OLD_LINKS}\""
"${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" > "${OLD_LINKS}"
cat "${OLD_LINKS}" | grep "${JIRAFEAU_OLDURL}" | while read REMOTE_LINK
cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK
do
REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
[ -z "${REMOTE_REF}" ] && continue
REMOTE_KEY=$(echo "${REMOTE_LINK}" | grep "k=" | sed 's%.*k=\([^&]*\).*%\1%')
# update periode for download
LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_URL}\" -t \"${JIRAFEAU_TIME}\" update \"${REMOTE_REF}\" 2>&1 >> \"${FIC_LOG}\""
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_URL}" -t "${JIRAFEAU_TIME}" update "${REMOTE_REF}" 2>&1 >> "${FIC_LOG}"
echo -n "/${REMOTE_REF}~${REMOTE_KEY}" >> "${PREV_CODES}"
LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -t \"${JIRAFEAU_TIME}\" update \"${REMOTE_REF}\" 2>&1 >> \"${FIC_LOG}\""
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -t "${JIRAFEAU_TIME}" update "${REMOTE_REF}" 2>&1 >> "${FIC_LOG}"
echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "${ARCHIVE_CONTENT}"
done
OLD_CODES=$(cat "${PREV_CODES}")
LOG_FIC " - OLD_CODES=${OLD_CODES}"
LOG_FIC " - archive starts with: $(cat ${ARCHIVE_CONTENT})"
# Etape extraction des pieces jointes
LOG_FIC "${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_DIR}/in.$$"
@ -124,46 +127,67 @@ LOG_FIC "${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_D
ATTACH_NAME=$(grep "^Name: " "${ATTACH_TMP_NAME}/meta" | cut -c 7- )
ATTACH_CONTENT_TYPE=$(grep "^Content-Type: " "${ATTACH_TMP_NAME}/meta" | cut -c 15- )
else
# XXX a virer
ATTACH_MEDIA="${ATTACH_TMP_NAME}"
ATTACH_NAME=$(basename "${ATTACH_MEDIA}")
ATTACH_CONTENT_TYPE=""
# XXX error
continue
fi
# Etape de televersement des pieces jointes
PASSWORD=$(apg -n 1 -m 12)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
actualSize=$(ls -l "${ATTACH_MEDIA}")
LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_URL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ATTACH_CONTENT_TYPE}\" -n \"${ATTACH_NAME}\" send \"${ATTACH_MEDIA}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\""
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_URL}" -s "${MAX_UPLOAD_SIZE}" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}"
LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ATTACH_CONTENT_TYPE}\" -n \"${ATTACH_NAME}\" send \"${ATTACH_MEDIA}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\""
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -s "${MAX_UPLOAD_SIZE}" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}"
cat "${ONE_LINK}" | {
read JIR_TOKEN
read JIR_CODE
LOG_FIC " - Jirafeau envoie ${JIR_TOKEN} et ${JIR_CODE}"
LOG_FIC " - Jirafeau retourne ${JIR_TOKEN} et ${JIR_CODE}"
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
LOG_FIC " - impossible de televerser ${ATTACH_TMP_FILE} (${JIR_TOKEN}), il ne sera pas remplace dans le message"
echo ""
echo "new:"
;;
* )
LOG_FIC " - substitution par ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
echo "${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5} /${JIR_TOKEN}~${PASSWORD_MD5}"
echo "${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}" >> "${ALL_LINKS}"
echo "url: ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "${ARCHIVE_CONTENT}"
;;
esac
}
LOG_FIC " - supprimer l'extraction ${ATTACH_TMP_FILE}"
rm -f "${ATTACH_TMP_FILE}"
done
# Création de l'archive
if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 4 ]; then
PASSWORD=$(apg -n 1 -m 12)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
LOG_FIC " - \"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"text/plain\" -n \"archive_content\" send \"${ARCHIVE_CONTENT}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\""
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -s "${MAX_UPLOAD_SIZE}" -c "text/plain" -n "archive_content" send "${ARCHIVE_CONTENT}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}"
cat "${ONE_LINK}" | {
read JIR_TOKEN
read JIR_CODE
LOG_FIC " - Jirafeau retourne ${JIR_TOKEN} et ${JIR_CODE}"
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
LOG_FIC " - impossible de televerser l'archive (${JIR_TOKEN}), il ne sera pas remplace dans le message"
echo "arch: bad"
;;
* )
LOG_FIC " - ajoute de l'archive ${JIRAFEAU_URL}/a.php?g=${JIR_TOKEN}~${PASSWORD_MD5}"
echo "arch: ${JIRAFEAU_URL}/a.php?g=${JIR_TOKEN}~${PASSWORD_MD5}"
;;
esac
}
else
LOG_FIC " - pas d'archive (moins de 2 PJ)"
echo "arch: none"
fi
# Etape de substitution
LOG_FIC "${SHRINK_CMD} -a \"${JIRAFEAU_URL}/t.php?n=${MAIL_SOURCE}_${DATE_TEMPS}&l=${OLD_CODES}\" -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\""
} | "${SHRINK_CMD}" -a "${JIRAFEAU_URL}/t.php?n=${MAIL_SOURCE}_${DATE_TEMPS}&l=${OLD_CODES}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
LOG_FIC "${SHRINK_CMD} -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\""
} | "${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
# XXX trace
# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" /var/mail/tmp/
# Etape choix de modification du message d'origine
if [ -s "${ALL_LINKS}" -o -s "${OLD_LINKS}" ]
then
if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 3 ]; then
# verification de taille finale
actualSize=$(wc -c < "${INSPECT_DIR}/in.$$.altered")
if [ ${actualSize} -ge $MAX_FINAL_SIZE ]; then

5
dockers/sympa/build.sh

@ -4,8 +4,13 @@ KAZ_ROOT=$(cd $(dirname $0)/../..; pwd)
. "${KAZ_ROOT}/bin/.commonFunctions.sh"
setKazVars
"${KAZ_BIN_DIR}/installDepollueur.sh"
printKazMsg "\n *** Création du Dockerfile Sympa"
cd "${KAZ_ROOT}"
mkdir -p dockers/postfix/filter/
rsync -a git/depollueur/src/bash/* git/depollueur/build/out/* dockers/sympa/filter/
docker build -t sympakaz . -f dockers/sympa/Dockerfile

8
dockers/sympa/docker-compose.yml

@ -1,6 +1,7 @@
version: '3.3'
services:
# la DB est-elle utile ?
db:
image: mariadb:10.5
container_name: ${sympaDBName}
@ -21,6 +22,9 @@ services:
restart: ${restartPolicy}
networks:
- sympaNet
- jirafeauNet
external_links:
- ${jirafeauServName}:${fileHost}
ports:
- ${SYMPA_IP}:25:25
#- ${SYMPA_IP}:143:143
@ -47,9 +51,9 @@ services:
- /etc/timezone:/etc/timezone:ro
#- /etc/ssl:/etc/ssl:ro
- /etc/letsencrypt:/etc/letsencrypt:ro
# environment:
environment:
- DOMAINNAME=${doamin}
# - HOSTNAME=${sympaHost}
# - DOMAINNAME=${sympaDomain}
# - CONTAINER_NAME=${sympaServName}
# env_file:
# - ../../secret/env-${sympaServName}

2
secret.tmpl/SetAllPass.sh

@ -152,12 +152,12 @@ roundcube_ROUNDCUBEMAIL_DB_TYPE="mysql"
roundcube_ROUNDCUBEMAIL_DB_NAME="${roundcube_MYSQL_DATABASE}"
roundcube_ROUNDCUBEMAIL_DB_USER="${roundcube_MYSQL_USER}"
roundcube_ROUNDCUBEMAIL_DB_PASSWORD="${roundcube_MYSQL_PASSWORD}"
roundcube_ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE="100Mo"
# XXX TODO >>>
# ROUNDCUBEMAIL_DB_PORT
# ROUNDCUBEMAIL_PLUGINS
# ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE
# ROUNDCUBEMAIL_SPELLCHECK_URI
# ROUNDCUBEMAIL_ASPELL_DICTS
# XXX TODO <<<

1
secret.tmpl/env-roundcubeServ

@ -3,3 +3,4 @@ ROUNDCUBEMAIL_DB_TYPE=
ROUNDCUBEMAIL_DB_NAME=
ROUNDCUBEMAIL_DB_USER=
ROUNDCUBEMAIL_DB_PASSWORD=
ROUNDCUBEMAIL_UPLOAD_MAX_FILESIZE=

Loading…
Cancel
Save