Petits détails dans les verifs

Makefile

@@ -71,7 +71,7 @@ $(OBJ_DIR)/%.o: $(SRC_DIR)/*/%.cpp
 	$(CC) $< $(IFLAGS) -cpp -c -o $@
 
 ## ENTRIES #############################
-all:	init eMailShrinker jirafeauAPI
+all:	init eMailShrinker jirafeauAPI doc
 
 eMailShrinker: $(KAZ_OUT)
 
@@ -83,6 +83,9 @@ jirafeauAPI: $(JIR_OUT)
 $(JIR_OUT): $(JIR_OBJ)
 	$(CC) $(JIR_OBJ) $(IFLAGS) -cpp -L$(LIB_DIR) $(LFLAGS) -o $@
 
+doc:
+	doxygen src/Doxyfile
+
 init:
 	mkdir -p $(OUT_DIR) $(OBJ_DIR) $(LIB_DIR)
 

README.md

@@ -33,7 +33,7 @@ depollueur/
 ## Compilation
 
 ```bash
-sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev
+sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev doxygen
 
 git clone https://git.kaz.bzh/KAZ/depollueur.git
 # or for contributors :
@@ -41,3 +41,14 @@ git clone https://git.kaz.bzh/KAZ/depollueur.git
 
 make -j $(nproc)
 ```
+
+##Astuces
+
+
+Pour la mise au point dans Kaz, il faut bloquer la mise à jour automatique avec :
+
+``` bash
+touch /kaz/git/depollueur/dontupdage
+/kaz/bin/container.sh stop jirafeau postfix; docker system prune; /kaz/bin/install.sh jirafeau postfix
+```
+

src/Jirafeau/a.php

@@ -12,6 +12,7 @@
  a.php?s=mel@domain.org&t=password + [action] => manage account
  action: a=login a=logout a=r[on|off] a=p[minute|hour|day|week|month|quarter]
 */
+
 use PHPMailer\PHPMailer\PHPMailer;
 use PHPMailer\PHPMailer\SMTP;
 use PHPMailer\PHPMailer\Exception;
@@ -24,9 +25,10 @@ require (JIRAFEAU_ROOT . 'lib/functions.php');
 require (JIRAFEAU_ROOT . 'lib/lang.php');
 
 define ('VAR_TOKENS', $cfg ['var_root'].'tokens/');
+define ('VAR_MODE', $cfg ['var_root'].'mode/');
 define ('VAR_TRACKS', $cfg ['var_root'].'tracks/');
-define ('VAR_LANG', $cfg ['var_root'].'lang/');
 define ('VAR_PERIOD', $cfg ['var_root'].'period/');
+define ('VAR_LANG', $cfg ['var_root'].'lang/');
 define ('VAR_FAKE', $cfg ['var_root'].'fake/');
 define ('VAR_ADMIN', $cfg ['var_root'].'admin/');
 
@@ -38,6 +40,7 @@ define ('MAX_VALID_UPLOAD_TIME', 60);
 define ('TOKEN_USE_LIMIT', "-2 hours");
 define ('TOKEN_LOGIN_LIMIT', "-15 minutes");
 define ('TOKEN_LOGOUT_LIMIT', "-8 hours");
+define ('DEFAULT_MODE', "footer");
 define ('DEFAULT_PERIOD', "month");
 define ('DEFAULT_LANG', "fr");
 
@@ -45,7 +48,7 @@ define ('E_BAD_ARCHIVE_NAME', 'Bad archive name format');
 define ('E_CREATE_ZIP', "Impossible de créer l'archive.");
 define ('E_OPEN_ZIP', "Impossible d'ouvrir l'archive.");
 define ('M_BAD_KEY', "Mauvaise clef pour ");
-define ('M_BAD_SENDER_NAME', 'Votre mèl est incorrect');
+define ('M_BAD_SENDER_NAME', 'Votre mél est incorrect');
 define ('M_BAD_TOKEN', "Vous n'utilisez pas le bon jeton (consultez votre messagerie).");
 define ('M_TOO_LONG_BEFORE_LOGGED', "Jeton de connexion trop ancien.");
 define ('M_TOO_LONG_LOGGED', "Temps de connexion dépassé.");
@@ -60,7 +63,7 @@ define ('M_FILES_RENAMED', " fichiers renommés.");
 define ('M_FILE_NOT_FOUND', " fichier est expiré.");
 define ('M_FILE_RENAMED', " fichier renommé.");
 define ('M_INTRO_FORM', "Quelles sont les informations me concernant ?");
-define ('M_MEL', "votre mèl");
+define ('M_MEL', "votre mél");
 define ('M_NO_FILENAME', 'SansNom');
 define ('M_NO_SENDER', 'kaz-');
 define ('M_NEW_ATTACHEMENT_DIRNAME', "nouveau");
@@ -74,13 +77,14 @@ define ('M_WELCOME', "<p>Informations concernant le compte : <b>___SENDER___</b>
 define ('M_INCONSISTENT_DATES',
         " (dates incoh&eacute;antes avec ___FILENAME___ : ___DIRTIME___ != ___FILETIME___)");
 
-define ('A_ACTION', 'a');			// action : T_LOGIN, T_LOGOUT, A_RECORD+(on|off), A_LANG(fr|en|br), A_PERIOD(minute|hour|day|week|month|quarter)
+define ('A_ACTION', 'a');			// action : T_LOGIN, T_LOGOUT, A_MODE(footer|attachment|both), A_RECORD+(on|off), A_PERIOD(minute|hour|day|week|month|quarter), A_LANG(fr|en|br)
 define ('A_GET', 'g');				// get archive
 define ('A_HASH', 'h');				// file to update or delete
 define ('A_OPEN_TOKEN', 'o');		// ask token
 define ('A_SENDER', 's');			// session sender
 define ('A_TOKEN', 't');			// session token
 define ('A_UPDATE', 'u');			// update perriod for file or archive
+define ('A_MODE', 'm');				// get mode status
 define ('A_RECORD', 'r');			// get track status
 define ('A_PERIOD', 'p');			// get period status
 define ('A_LANG', 'l');				// get lang status
@@ -110,7 +114,8 @@ define ('T_ARCHIVE_TITLE', "archive_content");
 define ('T_ARCHIVE_MIME', "text/kaz_email_archive");
 
 
-$langText = ['fr' => "Francais", 'br' => "Breton", 'en' => "english"];
+$modeText = ['footer' => "pied de page", 'attachment' => "pi&egrave;ce jointe", 'both' => "les deux"];
+$trackText = ['on' => "oui", 'off' => "non"];
 $periodText = ['minute' => "minute", 'hour' => "heure", 'day' => "jour", 'week' => "semaine", 'month' => "mois"];
 // XXX , 'quarter' => "trimestre"];
 $periodButton = ['hour' => ["&#128341;", ">1 heure"],
@@ -118,7 +123,7 @@ $periodButton = ['hour' => ["&#128341;", ">1 heure"],
                  'week' => ["&#128349;", "> 1 semaine"],
                  'month' => ["&#128350;", "> 1 mois"]];
 // XXX 'quarter' => ["&#128351;", "> 1 trimestre"]];
-$trackText = ['on' => "oui", 'off' => "non"];
+$langText = ['fr' => "Francais", 'br' => "Breton", 'en' => "english"];
 $doLogout = '';
 $message = '';
 
@@ -129,6 +134,19 @@ $message = '';
 /* Remove errors. */
 @error_reporting (0);
 
+// ========================================
+if (isset ($_REQUEST [A_MODE]) && !empty ($_REQUEST [A_MODE])) {
+    if (!preg_match ("/^([a-z0-9\+_\-]+)(\.[a-z0-9\+_\-]+)*@([a-z0-9\-]+\.)+[a-z]{2,6}$/i", $_REQUEST [A_MODE]))
+        $content = DEFAULT_MODE.NL;
+    else
+        $content = getSenderMode ($_REQUEST [A_MODE]).NL;
+    header ('HTTP/1.0 200 OK');
+    header ('Content-Length: ' . strlen ($content));
+    header ('Content-Type: text/plain');
+    echo $content;
+    exit;
+}
+
 // ========================================
 if (isset ($_REQUEST [A_RECORD]) && !empty ($_REQUEST [A_RECORD])) {
     if (!preg_match ("/^([a-z0-9\+_\-]+)(\.[a-z0-9\+_\-]+)*@([a-z0-9\-]+\.)+[a-z]{2,6}$/i", $_REQUEST [A_RECORD]))
@@ -155,6 +173,19 @@ if (isset ($_REQUEST [A_PERIOD]) && !empty ($_REQUEST [A_PERIOD])) {
     exit;
 }
 
+// ========================================
+if (isset ($_REQUEST [A_LANG]) && !empty ($_REQUEST [A_LANG])) {
+    if (!preg_match ("/^([a-z0-9\+_\-]+)(\.[a-z0-9\+_\-]+)*@([a-z0-9\-]+\.)+[a-z]{2,6}$/i", $_REQUEST [A_LANG]))
+        $content = DEFAULT_LANG.NL;
+    else
+        $content = getSenderLang ($_REQUEST [A_LANG]).NL;
+    header ('HTTP/1.0 200 OK');
+    header ('Content-Length: ' . strlen ($content));
+    header ('Content-Type: text/plain');
+    echo $content;
+    exit;
+}
+
 // ========================================
 $doUpdate = false;
 if (isset ($_REQUEST [A_UPDATE]) && !empty ($_REQUEST [A_UPDATE])) {
@@ -179,6 +210,29 @@ function returnError ($msg) {
     exit;
 }
 
+// ========================================
+function setSenderMode ($sender, $mode) {
+    if (!$sender)
+        return;
+    if (!file_exists (VAR_MODE))
+        mkdir (VAR_MODE, 0755);
+    if (empty ($mode) || DEFAULT_MODE == $mode) {
+        rmSenderMode ($sender);
+    } else
+        file_put_contents (VAR_MODE.$sender, $mode.NL);
+}
+function rmSenderMode ($sender) {
+    if (!$sender)
+        return;
+    if (file_exists (VAR_MODE.$sender))
+        unlink (VAR_MODE.$sender);
+}
+function getSenderMode ($sender) {
+    if ($sender && file_exists (VAR_MODE.$sender))
+        return trim (file (VAR_MODE.$sender)[0]);
+    return DEFAULT_MODE;
+}
+
 // ========================================
 function setSenderTrack ($sender) {
     if (!$sender)
@@ -197,29 +251,6 @@ function isSenderTrack ($sender) {
     return $sender && file_exists (VAR_TRACKS.$sender);
 }
 
-// ========================================
-function setSenderLang ($sender, $lang) {
-    if (!$sender)
-        return;
-    if (!file_exists (VAR_LANG))
-        mkdir (VAR_LANG, 0755);
-    if (empty ($lang) || DEFAULT_LANG == $lang) {
-        rmSenderLang ($sender);
-    } else
-        file_put_contents (VAR_LANG.$sender, $lang.NL);
-}
-function rmSenderLang ($sender) {
-    if (!$sender)
-        return;
-    if (file_exists (VAR_LANG.$sender))
-        unlink (VAR_LANG.$sender);
-}
-function getSenderLang ($sender) {
-    if ($sender && file_exists (VAR_LANG.$sender))
-        return trim (file (VAR_LANG.$sender)[0]);
-    return DEFAULT_LANG;
-}
-
 // ========================================
 function setSenderPeriod ($sender, $period) {
     if (!$sender)
@@ -272,6 +303,29 @@ function period2seconds ($periodName) {
     }
 }
 
+// ========================================
+function setSenderLang ($sender, $lang) {
+    if (!$sender)
+        return;
+    if (!file_exists (VAR_LANG))
+        mkdir (VAR_LANG, 0755);
+    if (empty ($lang) || DEFAULT_LANG == $lang) {
+        rmSenderLang ($sender);
+    } else
+        file_put_contents (VAR_LANG.$sender, $lang.NL);
+}
+function rmSenderLang ($sender) {
+    if (!$sender)
+        return;
+    if (file_exists (VAR_LANG.$sender))
+        unlink (VAR_LANG.$sender);
+}
+function getSenderLang ($sender) {
+    if ($sender && file_exists (VAR_LANG.$sender))
+        return trim (file (VAR_LANG.$sender)[0]);
+    return DEFAULT_LANG;
+}
+
 // ========================================
 function setSenderFake ($error, $sender, $owner, $dirLink, $fileLink) {
     global $doLogout;
@@ -748,10 +802,17 @@ if ($doDownload) {
                     $modif = true;
                     continue;
                 }
-                $srcName = $dstName = ($link ['file_name'] ? $link ['file_name'] : M_NO_FILENAME);
+                $dstName = ($link ['file_name'] ? $link ['file_name'] : M_NO_FILENAME);
-                if (in_array ($srcName, $singleName))
+                if (in_array ($dstName, $singleName)) {
-                    for ($i = 0; $i < 10000; ++$i) {
+                    $dstFilename = $dstName;
-                        $dstName = sprintf ("%s-%2d", $srcName, $i);
+                    $dstExtension = "";
+                    $dstExtensionPos = strrpos ($dstName, '.');
+                    if ($dstExtensionPos) {
+                        $dstFilename = substr ($dstName, 0, $dstExtensionPos);
+                        $dstExtension = substr ($dstName, $dstExtensionPos);
+                    }
+                    for ($i = 1; $i < 10000; ++$i) {
+                        $dstName = sprintf ("%s-%02d%s", $dstFilename, $i, $dstExtension);
                         if (!in_array ($dstName, $singleName)) {
                             if (isset ($archiveContent [T_RENAME]))
                                 ++$archiveContent [T_RENAME];
@@ -761,6 +822,7 @@ if ($doDownload) {
                             break;
                         }
                     }
+                }
                 $singleName [] = $dstName;
                 $archiveContent [$cat][T_ENTRIES][] = [T_HASH => $link ['hash'], T_FILENAME => $dstName, T_CRYPT_KEY => $cryptKey, T_CRYPTED => $link ['crypted']];
             }
@@ -871,7 +933,7 @@ if (isset ($_REQUEST [A_ACTION]) && $_REQUEST [A_ACTION] == T_LOGIN && $sender)
             echo M_SEND_TOKEN;
         else
             echo
-                "Erreur dans l'envoi. V&eacute;ritiez votre m&egrave;l.";
+                "Erreur dans l'envoi. V&eacute;ritiez votre m&eacute;l.";
     }
     echo "<br/><br/><br/>";
     require (JIRAFEAU_ROOT.'lib/template/footer.php');
@@ -954,6 +1016,10 @@ if ($doLogout || (isset ($_REQUEST [A_ACTION]) && $_REQUEST [A_ACTION] == T_LOGO
 if (isset ($_REQUEST [A_ACTION])) {
     // change track
     switch (true) {
+    case preg_match ("/^".A_MODE."(".implode ("|", array_keys ($modeText)).")$/i", $_REQUEST [A_ACTION], $matches):
+        setSenderMode ($sender, $matches [1]);
+        $message .= "Votre mode &agrave; &eacute;t&eacute; mise &agrave; jour.";
+        break;
     case preg_match ("/^".A_RECORD."(on|off)$/i", $_REQUEST [A_ACTION], $matches):
         if ($matches [1] == "on")
             setSenderTrack ($sender);
@@ -961,14 +1027,14 @@ if (isset ($_REQUEST [A_ACTION])) {
             rmSenderTrack ($sender);
         $message .= "Votre suivi &agrave; &eacute;t&eacute; mise &agrave; jour.";
         break;
-    case preg_match ("/^".A_LANG."(".implode ("|", array_keys ($langText)).")$/i", $_REQUEST [A_ACTION], $matches):
-        setSenderLang ($sender, $matches [1]);
-        $message .= "Votre lang &agrave; &eacute;t&eacute; mise &agrave; jour.";
-        break;
     case preg_match ("/^".A_PERIOD."(".implode ("|", array_keys ($periodText)).")$/i", $_REQUEST [A_ACTION], $matches):
         setSenderPeriod ($sender, $matches [1]);
         $message .= "Votre p&eacute;riode &agrave; &eacute;t&eacute; mise &agrave; jour.";
         break;
+    case preg_match ("/^".A_LANG."(".implode ("|", array_keys ($langText)).")$/i", $_REQUEST [A_ACTION], $matches):
+        setSenderLang ($sender, $matches [1]);
+        $message .= "Votre lang &agrave; &eacute;t&eacute; mise &agrave; jour.";
+        break;
     }
 }
 
@@ -1063,9 +1129,20 @@ div.frame {border: 1px; border-style: solid; padding: 1em; margin: 1em;}
 --></style>
 <?php
 $defaultChecked = [];
+$defaultChecked [getSenderMode ($sender)] = ' selected="selected"';
 $defaultChecked [isSenderTrack ($sender) ? "on" : "off"] = ' checked="checked"';
 $defaultChecked [getSenderPeriod ($sender)] = ' selected="selected"';
+$defaultChecked [getSenderLang ($sender)] = ' selected="selected"';
 echo
+    '<form method="post">'.
+    'Je veux que mes futurs envois soient d&eacute;pollu&eacute; en pla&ccedil;ant les liens de t&eacute;l&eacute;chargements '.
+    '<select name="'.A_ACTION.'" style="width: auto !important;">';
+foreach ($modeText as $item => $text)
+    echo '  <option value="'.A_MODE.$item.'"'.$defaultChecked [$item].'>'.$text.'</option>';
+echo
+    '</select> '.
+    '<button type="submit">'."valider".'</button>'.
+    '</form>'.
     '<form method="post">'.
     'Je veux que Kaz suive tous mes futurs envois: '.
     '<input type="hidden" name="'.A_SENDER.'" value="'.$sender.'"/>'.
@@ -1080,6 +1157,15 @@ echo
     '<select name="'.A_ACTION.'" style="width: auto !important;">';
 foreach ($periodText as $item => $text)
     echo '  <option value="'.A_PERIOD.$item.'"'.$defaultChecked [$item].'>'.$text.'</option>';
+echo
+    '</select> '.
+    '<button type="submit">'."valider".'</button>'.
+    '</form>'.
+    '<form method="post">'.
+    'Je préfère envoyer mes messages en langue '.
+    '<select name="'.A_ACTION.'" style="width: auto !important;">';
+foreach ($langText as $item => $text)
+    echo '  <option value="'.A_LANG.$item.'"'.$defaultChecked [$item].'>'.$text.'</option>';
 echo
     '</select> '.
     '<button type="submit">'."valider".'</button>'.

src/bash/filter.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 ##########################################################################
 # Copyright KAZ 2021							 #
 # 									 #
@@ -41,6 +41,10 @@
 #   le roriétaire du script
 ##########################################################################
 
+DEFAULT_MODE="footer"
+DEFAULT_PERIOD="month"
+DEFAULT_TRACK=""
+
 cd $(dirname $0)
 DOMAINNAME=$(cat domainname)
 # Exit codes from <sysexits.h>
@@ -61,11 +65,12 @@ JIRAFEAU_LOCAL=http://depot
 JIRAFEAU_TIME=month
 MD5_CMD=/usr/bin/md5sum
 DISCLAMER_CMD=altermime
-MAX_FINAL_SIZE=307200 # 300ki
+MAX_FINAL_SIZE=2097152 # 2Mi
 ARCHIVE_TITLE="archive_content"
 ARCHIVE_MIME="text/kaz_email_archive"
 
 KEEP_FAILED=true
+DEBUG=
 
 ####################  FONCTIONS ############################################
 BOLD=''
@@ -106,8 +111,14 @@ fi
 MAIL_SOURCE=$(echo $@ | awk 'BEGIN{FS=" "} {print $2}')
 DATE_TEMPS=$(date "+%Y-%m-%d-%H:%M:%S")
 REP_PIECE_JOINTE="${MAILS}/${DATE_TEMPS}_${MAIL_SOURCE}_$$"
-TRACK=$(curl "${JIRAFEAU_LOCAL}/a.php?r=${MAIL_SOURCE}" 2>/dev/null)
+
-PERIOD=$(curl "${JIRAFEAU_LOCAL}/a.php?p=${MAIL_SOURCE}" 2>/dev/null)
+MODE=$(curl "${JIRAFEAU_LOCAL}/a.php?m=${MAIL_SOURCE}" 2>/dev/null )
+[[ "${MODE}" =~ ^(footer|attachment|both)$ ]] || MODE="${DEFAULT_MODE}"
+TRACK=$(curl "${JIRAFEAU_LOCAL}/a.php?r=${MAIL_SOURCE}" 2>/dev/null )
+[[ "${TRACK}" =~ ^(|0|1|false|true|FALSE|TRUE|on|off)$ ]] || TRACK="${DEFAULT_TRACK}"
+PERIOD=$(curl "${JIRAFEAU_LOCAL}/a.php?p=${MAIL_SOURCE}" 2>/dev/null )
+[[ "${PERIOD}" =~ ^(minute|hour|day|week|month|quarter)$ ]] || PERIOD="${DEFAULT_PERIOD}"
+
 if [ -n "$(echo "${PERIOD}" | grep -e minute -e hour -e day -e week -e month -e quarter 2>/dev/null)" ]; then
     JIRAFEAU_TIME="${PERIOD}"
 fi
@@ -134,7 +145,7 @@ ARCHIVE_CONTENT="${REP_PIECE_JOINTE}/archive-content.txt"
 JIRAFEAU_ERROR="${REP_PIECE_JOINTE}/jirafeau-error.txt"
 
 # Clean up when done or when aborting.
-trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
+[ -z "${DEBUG}" ] && trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
 
 if ! cat > "in.$$"; then
     LOG_FIC "${RED}Cannot save mail to file${NC}"
@@ -142,8 +153,7 @@ if ! cat > "in.$$"; then
 fi
 LOG_FIC "\n" \
 	"             size: ${YELLOW}$(wc -c < "${INSPECT_DIR}/in.$$")${NC}"
-# XXX trace
+[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$" "${DIR_LOG}/pb/in.$$.orig")
-# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.bak"
 
 mkdir -p "${REP_PIECE_JOINTE}/"
 >"${OLD_LINKS}"
@@ -156,8 +166,7 @@ echo "time: ${DATE_TEMPS}\nid: $(date +%s)" > "${ARCHIVE_CONTENT}"
 LOG_FIC "${CYAN}${SHRINK_CMD} -u \"${INSPECT_DIR}/in.$$\" 2>> \"${FIC_LOG}\" > \"${OLD_LINKS}\"${NC}"
 "${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" > "${OLD_LINKS}"
 
-cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK
+cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
-do
     REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
     [ -z "${REMOTE_REF}" ] && continue
     REMOTE_KEY=$(echo "${REMOTE_LINK}" | grep "k=" | sed 's%.*k=\([^&]*\).*%\1%')
@@ -171,10 +180,8 @@ LOG_FIC " - archive starts with: ${NL}${YELLOW}$(cat ${ARCHIVE_CONTENT})${NC}"
 # Etape extraction des pieces jointes
 LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_DIR}/in.$$ ${NC}"
 "${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" -d "${REP_PIECE_JOINTE}" "${INSPECT_DIR}/in.$$"  2>> "${FIC_LOG}" | {
-    while read ATTACH_TMP_NAME
+    while read ATTACH_TMP_NAME; do
-    do
+	if [ -d "${ATTACH_TMP_NAME}" ]; then
-	if [ -d "${ATTACH_TMP_NAME}" ]
-	then
 	    ATTACH_MEDIA="${ATTACH_TMP_NAME}/media"
 	    ATTACH_NAME=$(grep "^Name: " "${ATTACH_TMP_NAME}/meta" | cut -c 7- )
 	    ATTACH_CONTENT_TYPE=$(grep "^Content-Type: " "${ATTACH_TMP_NAME}/meta" | cut -c 15- )
@@ -195,7 +202,7 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
 	    case "${JIR_TOKEN}" in
 		"" | no | *Error* | \<* )
 		    LOG_FIC " - ${RED}Can't upload <${ATTACH_NAME}> <${ATTACH_CONTENT_TYPE}> <$(wc -c < "${ATTACH_MEDIA}")> (${JIR_TOKEN}) <in.$$.bak>. It will be not change in e-mail.${NC}"
-		    echo "new:"
+		    echo "url:"
 		    keepFailed "${INSPECT_DIR}/in.$$"
 		    echo "UPLOAD_FAIL" >> "${JIRAFEAU_ERROR}"
 		    ;;
@@ -240,11 +247,10 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
 	echo "arch: none"
     fi
     # Etape de substitution
-    LOG_FIC "${CYAN}${SHRINK_CMD} -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"${NC}"
+    LOG_FIC "${CYAN}${SHRINK_CMD} -m \"${MODE}\" -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"${NC}"
-} | "${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
+} | "${SHRINK_CMD}" -m "${MODE}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
 
-# XXX trace
+[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$.altered" "${DIR_LOG}/pb/in.$$.altered")
-# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" /var/mail/tmp/
 
 if [ -s "${JIRAFEAU_ERROR}" ]; then
     LOG_FIC " - ${RED}upload fail${NC}"

src/bash/filterTest.sh

@@ -0,0 +1,136 @@
+#!/bin/bash
+
+PRG=$(basename $0)
+
+ATTACH_MODE="-m BOTH"
+
+BOLD=''
+RED=''
+GREEN=''
+YELLOW=''
+BLUE=''
+MAGENTA=''
+CYAN=''
+NC='' # No Color
+NL='
+'
+
+TTY=$(tty)
+
+LOG () {
+    echo "$1" >> "${TTY}"
+}
+
+usage () {
+    echo "Usage: ${PRG} mbox"
+    exit 1
+}
+
+[ "$#" -eq 1 ] || usage
+
+mbox=$(realpath $1)
+
+
+cd $(dirname $0)
+DOMAINNAME="$(cat domainname)"
+JIRAFEAU_URL="https://depot.${DOMAINNAME}"
+JIRAFEAU_LOCAL="${JIRAFEAU_URL}"
+
+cd ../..
+
+mkdir -p tmp
+rm -fr tmp/PJ-name.txt tmp/PJ-Keys.txt tmp/PJ tmp/archive-content.txt tmp/url-to-refresh.txt tmp/new-mbox
+echo -e "time: $(date "+%Y-%m-%d-%H:%M:%S")\nid: $(date +%s)" > "tmp/archive-content.txt"
+
+LOG "  - ${BLUE}mbox: ${mbox}${NC}"
+build/out/eMailShrinker -l "${mbox}"
+LOG
+
+build/out/eMailShrinker -u "${mbox}" > "tmp/url-to-refresh.txt" 2>> "${TTY}"
+
+
+cat "tmp/url-to-refresh.txt" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
+    REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
+    [ -z "${REMOTE_REF}" ] && continue
+    LOG "  - ${BLUE}update ${REMOTE_REF}${NC}"
+    build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" update "${REMOTE_REF}" 2>> "${TTY}"
+    LOG
+    echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "tmp/archive-content.txt"
+done
+
+build/out/eMailShrinker -s "5ki" -d tmp/PJ "${mbox}" > "tmp/PJ-name.txt"
+
+LOG "  - ${BLUE}PJ-name: ${NC}"
+cat tmp/PJ-name.txt
+LOG
+
+cat "tmp/PJ-name.txt" | {
+    while read ATTACH_TMP_NAME; do
+	LOG " - ${BLUE}find ATTACH_TMP_NAME: (${ATTACH_TMP_NAME}) ${NC}"
+	if [ -d "${ATTACH_TMP_NAME}" ]; then
+	    ATTACH_MEDIA="${ATTACH_TMP_NAME}/media"
+	    ATTACH_NAME=$(grep "^Name: " "${ATTACH_TMP_NAME}/meta" | cut -c 7- )
+	    ATTACH_CONTENT_TYPE=$(grep "^Content-Type: " "${ATTACH_TMP_NAME}/meta" | cut -c 15- )
+	else
+	    LOG " - ${RED}no ATTACH_TMP_NAME: ${ATTACH_TMP_NAME}${NC}"
+	    # XXX error
+	    continue
+	fi
+	LOG " - ${BLUE}find ${ATTACH_NAME} / (${ATTACH_CONTENT_TYPE}) / ${ATTACH_MEDIA} ${NC}"
+	PASSWORD=$(apg -n 1 -m 12)
+	PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \  -f 1)
+	build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" -s "1Gi" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}"  2>> "${TTY}" > "tmp/one.txt"
+
+	cat "tmp/one.txt"  | {
+	    read JIR_TOKEN
+	    read JIR_CODE
+	    case "${JIR_TOKEN}" in
+		"" | no | *Error* | \<* )
+		    LOG "  - ${RED}can't upload ${ATTACH_MEDIA} <${JIR_TOKEN}> <${JIR_CODE}>${NC}"
+		    cat "tmp/one.txt" >> "${TTY}"
+		    echo "url:"
+		    ;;
+		* )
+		    LOG "  - ${GREEN} upload ${ATTACH_MEDIA}${NC}"
+		    echo "url: ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
+		    echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "tmp/archive-content.txt"
+		    ;;
+	    esac
+	}
+    done
+    NB_ATTACH=$(grep -e "^old: " -e "^new: " "tmp/archive-content.txt" | wc -l)
+    if [ "${NB_ATTACH}" -gt 1 ]; then
+	PASSWORD=$(apg -n 1 -m 12)
+	PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \  -f 1)
+	build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" -s "1Gi" -c "text/kaz_email_archive" -n "archive_content" send "tmp/archive-content.txt" "${PASSWORD}" > "tmp/one.txt"  2>> "${TTY}"
+	cat "tmp/one.txt"  | {
+	    read JIR_TOKEN
+	    read JIR_CODE
+	    case "${JIR_TOKEN}" in
+		"" | no | *Error* | \<* )
+		    LOG "  - ${RED}can't upload tmp/archive-content.txt${NC}"
+		    echo "arch: bad"
+		    ;;
+		* )
+		    LOG "  - ${GREEN} upload archive-content.txt${NC}"
+		    echo "arch: ${JIRAFEAU_URL}/a.php?g=${JIR_TOKEN}~${PASSWORD_MD5}"
+		    ;;
+	    esac
+	}
+    else
+	LOG "  - ${GREEN} no archive${NC}"
+	echo "arch: none"
+    fi
+} > "tmp/PJ-Keys.txt"
+
+LOG "  - ${BLUE}PJ-Keys: ${NC}"
+cat tmp/PJ-Keys.txt
+LOG
+
+LOG "  - ${GREEN}ATTACH_MODE: ${ATTACH_MODE}${NC}"
+
+cat "tmp/PJ-Keys.txt" | build/out/eMailShrinker ${ATTACH_MODE} -s "5ki" "${mbox}" "tmp/new-mbox" 2>> "${TTY}"
+
+
+LOG "  - ${BLUE}new-mbox:${NC}"
+build/out/eMailShrinker -l "tmp/new-mbox" 2>> "${TTY}"

src/cpp/Attachment.cpp

@@ -61,11 +61,19 @@ const string Attachment::PLAIN				("plain");
 const string Attachment::HTML				("html");
 const string Attachment::RELATED			("related");
 const string Attachment::ALTERNATIVE			("alternative");
+const string Attachment::KAZ_ATTACH_NAME		(".---KazAttachment---.html");
 
 
 const regex Attachment::nameCharsetRegEx		(".*name\\*=(.*)");
-const regex Attachment::nameRegEx			(".*name=\"([^\"]*)\".*");
+const regex Attachment::nameRegEx			(".*name=\\s*\"?(.*)\"?\\s*;?\\s*");
-const regex Attachment::boundaryRegEx			(".*boundary=\"?([^\" ]*)\"?.*");
+// boundary="----=_Part_796779_1154936629.1668080348646"
+// boundary="------------040709000505010508040808"
+// boundary="----------=_1668606031-941125-91"
+// boundary="_004_PAVPR10MB6792713B313048E3A259B215B2079PAVPR10MB6792EURP_";
+// boundary="_000_PAVPR10MB6792713B313048E3A259B215B2079PAVPR10MB6792EURP_"
+// boundary=--boundary_1351_64006126-2b0e-4a3b-98ac-4797d1634188
+// boundary=--boundary_1352_7e294c9a-cfab-44a0-bfb3-7310380ac7cb;
+const regex Attachment::boundaryRegEx			(".*boundary=\"?([^\"; ]*)\"?;?.*");
 const regex Attachment::cidDefRegEx			(".*<([^>]*)>.*");
 const regex Attachment::textRegEx			(".*text/("+PLAIN+"|"+HTML+").*");
 const regex Attachment::multiRegEx			("\\s*multipart/(mixed|"+RELATED+"|"+ALTERNATIVE+").*");
@@ -266,6 +274,7 @@ Attachment::Attachment (ifstream &mbox, const int &level, const streamoff beginI
     toExtract (false),
     toUpdate (false),
     toDisclaim (false),
+    isKazAttachment (false),
     boundaryMiddleSize (0) {
   DEF_LOG ("Attachment::Attachment", "curPos: " << curPos << " level: " << level);
   readMime (mbox, curPos);
@@ -354,7 +363,11 @@ Attachment::nextBondary (ifstream &mbox, streamoff &curPos) {
 	   ++it)
 	if (couple.find (*it) != string::npos) {
 	  LOG ("find: "+ *it);
-	  subAttachements.back ().toUpdate = true;
+	  LOG ("size: " << subAttachements.size ());
+	  if (subAttachements.size ())
+	    subAttachements.back ().toUpdate = true;
+	  else
+	    LOG_BUG (true, continue, "eMailShrinker: bug A10: boundary format ? " << *this);
 	}
       prev = clearLine;
       continue;
@@ -406,17 +419,21 @@ Attachment::markSignificant (const string &parentMultiProp, const streamoff &min
   for (Attachment &sub : subAttachements)
     cantBeExtract |= sub.markSignificant (multiProp, minAttachSize, mbox, allMarkedPtrs);
   if (getProp (contentTypeToken, textRegEx) == HTML) {
-    string content = getContent (mbox);
+    if (KAZ_ATTACH_NAME == getAttachName ()) {
-    vector<string> imgs;
+      isKazAttachment = true;
-    getSection (content, IMG_BEGIN, IMG_END, imgs);
+    } else {
-    EmbeddedData::fillEmbeddedData (imgs, minAttachSize, embeddedData);
+      string content = getContent (mbox);
-    if (embeddedData.size ())
+      vector<string> imgs;
-      toUpdate = true;
+      getSection (content, IMG_BEGIN, IMG_END, imgs);
+      EmbeddedData::fillEmbeddedData (imgs, minAttachSize, embeddedData);
+      if (embeddedData.size ())
+	toUpdate = true;
+    }
   }
   cantBeExtract |= toUpdate;
   if (boundary.empty () && getSize () >= minAttachSize && !cantBeExtract)
     cantBeExtract = toExtract = true; // XXX cantBeExtract ?
-  if (toExtract || toUpdate || toDisclaim)
+  if (toExtract || toUpdate || toDisclaim || isKazAttachment)
     allMarkedPtrs.push_back (this);
   return cantBeExtract;
 }
@@ -489,7 +506,7 @@ kaz::operator << (ostream& os, const Attachment& attachment) {
   if (prop.length ())
     prop = " ["+prop+"]";
   
-  os << setw ((attachment.level % 20)*2) << "" << setw (10) << SizeArg (attachment.getSize ()) << "          " << attachment.getContentType ()
+  os << ("****************************************"+40-(attachment.level % 20)*2) << setw (10) << SizeArg (attachment.getSize ()) << "          " << attachment.getContentType ()
      << prop << (attachment.cid.length () ? " id: "+attachment.cid : "")
      << (attachment.boundary.length () ? " boundary: "+attachment.boundary : "")
      << " (" << attachment.beginPos << " / " << attachment.contentPos << " / " << attachment.endPos << ") " << endl;

src/cpp/MainAttachment.cpp

@@ -32,14 +32,18 @@
 // knowledge of the CeCILL-B license and that you accept its terms.	  //
 ////////////////////////////////////////////////////////////////////////////
 
-#include <iostream>
+#include <algorithm>
-#include <vector>
-#include <set>
 #include <fstream>
 #include <iomanip>
+#include <iostream>
 #include <math.h>
-#include <algorithm>
+#include <regex>
+#include <set>
 #include <unistd.h>
+#include <vector>
+
+#include <boost/assign.hpp>
+#include <boost/algorithm/string.hpp>
 
 #include "kazDebug.hpp"
 #include "kazMisc.hpp"
@@ -50,64 +54,108 @@
 using namespace std;
 using namespace kaz;
 
-static const string KAZ_WEB_SITE		= "https://kaz.bzh/";
+static const string KAZ_WEB_SITE		("https://kaz.bzh/");
-static const string TMPL_DOWNLOAD		= "{{DOWNLOAD}}";
+static const string TMPL_DOWNLOAD		("{{DOWNLOAD}}");
-static const string TMPL_FILENAME		= "{{FILENAME}}";
+static const string TMPL_FILENAME		("{{FILENAME}}");
-static const string CID				= "cid:";
+static const string CID				("cid:");
 
-static const string KAZ_PLAIN_HR		= "______________________________________________________________________________";
+// "l=/" => v1 compatibility
-static const string KAZ_PLAIN_START		= "~~ PJ-KAZ !"; // don't end whith space
+static const regex archiveURLSignature		(".*(([&?]g=)|([&?]l=/)).*");
-static const string KAZ_PLAIN_STOP		= KAZ_PLAIN_START+" ~~";
-static const string KAZ_PLAIN_DONT_TOUCH	= "(conservez cette partie intacte dans votre réponse si vous voulez transmettre les documents précédents)";
-static const string KAZ_PLAIN_WARNING		= "Attention : Kaz a dépollué ce message. Les pièces jointes ont été retirées et placées dans un dépôt provisoire. Elles seront automatiquement supprimées dans 1 mois. Si elles sont importantes et que vous souhaitez les conserver, vous devez utiliser les liens ci-dessous. Pour mieux comprendre la politique de nos services visitez kaz.bzh";
-static const string KAZ_PLAIN_DOWLOAD_ONE	= "Vos pièces jointes sont à télécharger individuellement ici :";
-static const string KAZ_PLAIN_DOWLOAD_OTHER	= "(Contenu dans des messages précédents)";
-static const string KAZ_PLAIN_DOWLOAD_ALL	= "Vous pouvez télécharger l'ensemble dans une archive là :";
 
-static const string HEAD			= "<head>";
-static const string HEAD_END			= "</head>";
-static const string KAZ_CSS_URL			= "https://kaz.bzh/m/email.css";
-static const string KAZ_CSS			= "<link rel=\"stylesheet\" type=\"text/css\" charset=\"utf-8\" href=\""+KAZ_CSS_URL+"\"/>";
-static const string A_END			= "</a>";
-static const string LI_BEGIN			= "<li";
-static const string CLASS_ONE			= "class=\"one\"";
-static const string LI_ONE			= LI_BEGIN+" "+CLASS_ONE+">";
-static const string LI_ALL			= LI_BEGIN+" class=\"all\">";
-static const string LI_END			= "</li>";
-static const string HREF_ONE			= "href=\"";
-static const string BODY_END			= "</body>";
-static const string HTML_END			= "</html>";
 
-static const string KAZ_HTML_TAG		= "<!--KAZ"; // don't end whith space
+static const string KAZ_PLAIN_HR		("______________________________________________________________________________");
-static const string KAZ_HTML_START		= KAZ_HTML_TAG+" START-->";
+static const string KAZ_PLAIN_START		("~~ PJ-KAZ !"); // don't end whith space
-static const string KAZ_HTML_STOP		= KAZ_HTML_TAG+" STOP-->";
+static const string KAZ_PLAIN_STOP		(KAZ_PLAIN_START+" ~~");
+static const string KAZ_PLAIN_DONT_TOUCH	("(conservez cette partie intacte dans votre réponse si vous voulez transmettre les documents précédents)");
+static const string KAZ_PLAIN_WARNING	        ("Attention : Kaz a dépollué ce message. Les pièces jointes ont été retirées et placées dans un dépôt provisoire. "
+						 "Elles seront automatiquement supprimées dans 1 mois. "
+						 "Si elles sont importantes et que vous souhaitez les conserver, vous devez utiliser les liens ci-dessous. "
+						 "Pour mieux comprendre la politique de nos services visitez kaz.bzh");
+static const string KAZ_PLAIN_DOWLOAD_ONE	("Vos pièces jointes sont à télécharger individuellement ici :");
+static const string KAZ_PLAIN_DOWLOAD_OTHER	("(Contenu dans des messages précédents)");
+static const string KAZ_PLAIN_DOWLOAD_ALL	("Vous pouvez télécharger l'ensemble dans une archive là :");
+
+static const string HEAD			("<head>");
+static const string HEAD_END			("</head>");
+static const string KAZ_CSS_URL			("https://kaz.bzh/m/email.css");
+static const string KAZ_CSS			("<link rel=\"stylesheet\" type=\"text/css\" charset=\"utf-8\" href=\""+KAZ_CSS_URL+"\"/>");
+static const string A_END			("</a>");
+static const string LI_BEGIN			("<li");
+static const string CLASS_ONE			("class=\"one\"");
+static const string LI_ONE			(LI_BEGIN+" "+CLASS_ONE+">");
+static const string LI_ALL			(LI_BEGIN+" class=\"all\">");
+static const string LI_END			("</li>");
+static const string HREF_ONE			("href=\"");
+static const string KAZ_HTML_CONTENT		("<!DOCTYPE html><html lang=\"fr\"><head><meta charset=\"utf-8\"><title>KAZ</title>"+KAZ_CSS+"</head><body>");
+static const string BODY_END			("</body>");
+static const string HTML_END			("</html>");
+
+static const string KAZ_HTML_TAG		("<!--KAZ"); // don't end whith space
+static const string KAZ_HTML_START		(KAZ_HTML_TAG+" START-->");
+static const string KAZ_HTML_STOP		(KAZ_HTML_TAG+" STOP-->");
 // Textes précédents encodés en SGML
-static const string KAZ_HTML_DONT_TOUCH		= "(conservez cette partie intacte dans votre r&eacute;ponse si vous voulez transmettre les documents pr&eacute;c&eacute;dents)";
+static const string KAZ_HTML_DONT_TOUCH		("(conservez cette partie intacte dans votre r&eacute;ponse si vous voulez transmettre les documents pr&eacute;c&eacute;dents)");
-static const string KAZ_HTML_DOWLOAD_ONE	= "Vos pi&egrave;ces jointes sont &agrave; t&eacute;l&eacute;charger individuellement ici :";
+static const string KAZ_HTML_DOWLOAD_ONE	("Vos pi&egrave;ces jointes sont &agrave; t&eacute;l&eacute;charger individuellement ici :");
-static const string KAZ_HTML_DOWLOAD_OTHER	= "(Contenu dans des messages pr&eacute;c&eacute;dents)";
+static const string KAZ_HTML_DOWLOAD_OTHER      ("(Contenu dans des messages pr&eacute;c&eacute;dents)");
-static const string KAZ_HTML_DOWLOAD_ALL	= "Vous pouvez t&eacute;l&eacute;charger l'ensemble dans une archive l&agrave; :";
+static const string KAZ_HTML_DOWLOAD_ALL	("Vous pouvez t&eacute;l&eacute;charger l'ensemble dans une archive l&agrave; :");
-static const string KAZ_HTML_ARCHIVE		= "archive";
+static const string KAZ_HTML_ARCHIVE		("archive");
+
+static const string KAZ_EMPTY_TEXT_PLAIN	("Content-Type: text/plain; charset=utf-8\n"
+						 "Content-Transfer-Encoding: base64\n");
+static const string KAZ_ATTACHMENT_TEXT_HTML	("Content-Type: text/html; charset=utf-8\n"
+						 "Content-Disposition: attachment; filename=\"" + Attachment::KAZ_ATTACH_NAME + "\"\n"
+						 "Content-Transfer-Encoding: base64\n");
+
 
 // ================================================================================
 vector <string>
 Attachment::stringsToUpdate ({KAZ_PLAIN_START, "\""+CID});
 
 // ================================================================================
-const string MainAttachment::templatePlainAddLink (" - "+TMPL_DOWNLOAD+" "+TMPL_FILENAME+"\r\n");
+const string MainAttachment::templatePlainAddLink	("  * "+TMPL_FILENAME+" <"+TMPL_DOWNLOAD+">\r\n");
-const string MainAttachment::templatePlainAllLink ("\r\n"+KAZ_PLAIN_DOWLOAD_ALL+"\r\n * "+TMPL_DOWNLOAD+"\r\n");
+const string MainAttachment::templatePlainAllLink	("\r\n  * "+KAZ_PLAIN_DOWLOAD_ALL+" <"+TMPL_DOWNLOAD+">\r\n");
 
-const string MainAttachment::templateHtmlHeader (KAZ_HTML_START+"<p style=\"clear: left; padding: 1pc 0 0 0; font-size:10px; color:#969696;\">"+KAZ_PLAIN_START+"</p><hr>\n"
+const string MainAttachment::templateHtmlHeader		(KAZ_HTML_START+"<p style=\"clear: left; padding: 1pc 0 0 0; font-size:10px; color:#969696;\">"+KAZ_PLAIN_START+"</p><hr>\n"
-						 "<div class=\"kaz\">"
+							 "<div class=\"kaz\">"
-						 "<p style=\"font-size:10px; color:#969696;\">"+KAZ_HTML_DONT_TOUCH+"</p>\n"
+							 "<p style=\"font-size:10px; color:#969696;\">"+KAZ_HTML_DONT_TOUCH+"</p>\n"
-						 "<p>"+KAZ_HTML_DOWLOAD_ONE+"<ul>\n");
+							 "<p>"+KAZ_HTML_DOWLOAD_ONE+"<ul>\n");
-const string MainAttachment::templateHtmlAddLink (LI_ONE+"<a "+HREF_ONE+TMPL_DOWNLOAD+"\">"+TMPL_FILENAME+"</a>"+LI_END+"\n");
+const string MainAttachment::templateHtmlAddLink	(LI_ONE+"<a "+HREF_ONE+TMPL_DOWNLOAD+"\">"+TMPL_FILENAME+"</a>"+LI_END+"\n");
-const string MainAttachment::templateHtmlOtherLink ("</ul>"+KAZ_HTML_DOWLOAD_OTHER+"<ul>\n");
+const string MainAttachment::templateHtmlOtherLink	("</ul>"+KAZ_HTML_DOWLOAD_OTHER+"<ul>\n");
-const string MainAttachment::templateHtmlAllLink ("</ul><ul>"+LI_ALL+KAZ_HTML_DOWLOAD_ALL+" <a href=\""+TMPL_DOWNLOAD+"\">"+KAZ_HTML_ARCHIVE+"</a>"+LI_END+"\n");
+const string MainAttachment::templateHtmlAllLink	("</ul><ul>"+LI_ALL+KAZ_HTML_DOWLOAD_ALL+" <a href=\""+TMPL_DOWNLOAD+"\">"+KAZ_HTML_ARCHIVE+"</a>"+LI_END+"\n");
-const string MainAttachment::templateHtmlFooter ("</ul></p>\n"
+const string MainAttachment::templateHtmlFooter		("</ul></p>\n"
-						 "<p class=\"msg\"><a class=\"kaz\" href=\""+KAZ_WEB_SITE+"\"> "+KAZ_WEB_SITE+" </a></p></div>\n"
+							 "<p class=\"msg\"><a class=\"kaz\" href=\""+KAZ_WEB_SITE+"\"> "+KAZ_WEB_SITE+" </a></p></div>\n"
-						 "<hr><p style=\"font-size:10px; color:#969696;\">"+KAZ_PLAIN_STOP+"</p>"+KAZ_HTML_STOP+"\n");
+							 "<hr><p style=\"font-size:10px; color:#969696;\">"+KAZ_PLAIN_STOP+"</p>"+KAZ_HTML_STOP+"\n");
 
-const regex MainAttachment::whiteSpaceRegEx ("\\s+");
+const regex MainAttachment::whiteSpaceRegEx		("\\s+");
+
+// ================================================================================
+const string
+kaz::attachModeLabels[] = {
+  "None", "Footer", "Attachment", "Both"
+};
+const map<string, AttachMode>
+kaz::attachModeMap = boost::assign::map_list_of
+  ("none", NONE)
+  ("footer", FOOTER)
+  ("attachment", ATTACHMENT)
+  ("both", BOTH)
+  ;
+ostream &
+kaz::operator << (ostream &out, const AttachMode &attachMode) {
+  //BOOST_ASSERT (treeType >= MIN && treeType <= ALPHA);
+  return out << attachModeLabels [attachMode];
+}
+istream &
+kaz::operator >> (istream &in, AttachMode &attachMode) {
+  string token;
+  in >> token;
+  auto pos = attachModeMap.find (boost::algorithm::to_lower_copy (token));
+  if (pos == attachModeMap.end ())
+    in.setstate (ios_base::failbit);
+  else
+    attachMode = pos->second;
+  return in;
+}
 
 // ================================================================================
 void
@@ -167,7 +215,8 @@ MainAttachment::addLink (string &plain, string &html, const string &url, const s
   plain += plainNewOneLink;
   string htmlNewOneLink (templateHtmlAddLink);
   string codedUrl (url);
-  replaceAll (codedUrl, "&", "&amp;");
+  // pb &amp;
+  // replaceAll (codedUrl, "&", "&amp;");
   replaceAll (htmlNewOneLink, TMPL_DOWNLOAD, codedUrl);
   replaceAll (htmlNewOneLink, TMPL_FILENAME, name);
   html += htmlNewOneLink;
@@ -208,7 +257,7 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
     return;
   }
 
-  plain = "\r\n"+KAZ_PLAIN_START+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_DONT_TOUCH+"\r\n\r\n"+KAZ_PLAIN_WARNING+"\r\n\r\n"+KAZ_PLAIN_DOWLOAD_ONE+"\r\n"+plainNewLinks;
+  plain = "\r\n"+KAZ_PLAIN_START+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_DONT_TOUCH+"\r\n\r\n"+KAZ_PLAIN_DOWLOAD_ONE+"\r\n"+plainNewLinks;
   html = templateHtmlHeader+htmlNewLinks;
   if (previousLinks.size ()) {
     plain += "\r\n"+KAZ_PLAIN_DOWLOAD_OTHER+"\r\n"+plainOldLinks;
@@ -218,12 +267,12 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
     string allPlainLinks (templatePlainAllLink);
     replaceAll (allPlainLinks, TMPL_DOWNLOAD, archiveDownloadURL);
     plain += allPlainLinks;
-    string allLinks (templateHtmlAllLink);
+    string allHtmlLinks (templateHtmlAllLink);
-    replaceAll (allLinks, TMPL_DOWNLOAD, archiveDownloadURL);
+    replaceAll (allHtmlLinks, TMPL_DOWNLOAD, archiveDownloadURL);
-    html += allLinks;
+    html += allHtmlLinks;
   }
-  html += templateHtmlFooter;
+  html += templateHtmlFooter+"\r\n";
-  plain += "\r\n\r\n"+KAZ_WEB_SITE+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_STOP+"\r\n";
+  plain += "\r\n\r\n"+KAZ_WEB_SITE+"\r\n\r\n"+KAZ_PLAIN_WARNING+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_STOP+"\r\n";
   // & => &amp; done
   LOG ("plain: " << plain);
   LOG ("html: " << html);
@@ -245,6 +294,7 @@ MainAttachment::addPrevious (const string &href, const string &name, const bool
 
 void
 MainAttachment::extractLinks (const string &extractedPlainKAZ) {
+  // plain text => "* name <url>"
   DEF_LOG ("Attachment::extractedPlainKAZ", "extractedPlainKAZ: " << extractedPlainKAZ);
   for (string::size_type startPos (0);
        (startPos = extractedPlainKAZ.find ("http", startPos)) != string::npos;
@@ -257,22 +307,30 @@ MainAttachment::extractLinks (const string &extractedPlainKAZ) {
 
     if (extractedPlainKAZ [stopPos])
       ++stopPos;
+    streamoff stopName = startPos;
     startPos = stopPos;
+
     // get all href but KAZ_WEB_SITE
     // the archive link will be skip by filter.sh
     if (href == KAZ_WEB_SITE)
       continue;
 
-    string::size_type nextPos = extractedPlainKAZ.find ("http", startPos);
+    // backward find name
-    string name (extractedPlainKAZ, startPos, (nextPos == string::npos ? extractedPlainKAZ.length () : nextPos) - startPos);
+    string::size_type startName = extractedPlainKAZ.rfind ("* ", stopName);
-    // skip [> \r\n\t]
+    string name;
-    nextPos = name.find_first_not_of ("[> \t\r\n\"]");
+
-    if (nextPos != string::npos)
+    if (startName != string::npos) {
-      name.erase (0, nextPos);
+      name = string (extractedPlainKAZ, startName+3, stopName - startName - 3);
-    // stop before [>\r\n\t]
+      // skip [> \r\n\t]
-    nextPos = name.find_first_of ("[\"]");
+      string::size_type nextPos = name.find_first_not_of ("[>< \t\r\n\"]");
-    if (nextPos != string::npos)
+      if (nextPos != string::npos)
-      name.erase (nextPos);
+	name.erase (0, nextPos);
+      // stop before [>\r\n\t]
+      nextPos = name.find_first_of ("[\"<]");
+      if (nextPos != string::npos)
+	name.erase (nextPos);
+    }
+
     LOG ("plain name: " << name);
     addPrevious (href, name);
   }
@@ -281,6 +339,7 @@ MainAttachment::extractLinks (const string &extractedPlainKAZ) {
 // ================================================================================
 void
 MainAttachment::extractLinks (const vector<string> &liOne) {
+  // html text => "<li ...><a href="url">name</a>"
   DEF_LOG ("Attachment::extractedPlainKAZ", "liOne.size: " << liOne.size ());
   for (const string &one : liOne) {
     if (caseInsensitiveFind (one, CLASS_ONE) == string::npos)
@@ -317,7 +376,7 @@ MainAttachment::extractPreviousKAZ (ifstream &mbox) {
   DEF_LOG ("MainAttachment::extractPreviousKAZ", "");
   string extractedPlainKAZ, extractedHtmlKAZ;
   for (const Attachment *attachP : allMarkedPtrs) {
-    if (!attachP->toUpdate || isBase64Encoding ())
+    if (!(attachP->toUpdate || attachP->isKazAttachment)) // isKazAttachment => toUpdate
       continue;
     string textProp = attachP->getProp (contentTypeToken, textRegEx);
     if (textProp.empty ())
@@ -353,17 +412,20 @@ MainAttachment::extractPreviousKAZ (ifstream &mbox) {
 void
 MainAttachment::removePreviousArchive () {
   vector<string> toRemove;
-  for (map <string, string>::const_iterator it = previousLinks.begin (); it != previousLinks.end (); ++it)
+  for (map <string, string>::const_iterator it = previousLinks.begin (); it != previousLinks.end (); ++it) {
-    if (it->first.find ("&g=") != string::npos ||
+    const string key (it->first);
-	it->first.find ("&l=/") != string::npos) // v1 compatibility
+    if (regex_match (key, archiveURLSignature))
-      toRemove.push_back (it->first);
+      toRemove.push_back (key);
+  }
   for (string old : toRemove)
     previousLinks.erase (old);
 }
 
 // ================================================================================
 MainAttachment::MainAttachment (ifstream &mbox)
-  : Attachment (mbox, initTmpLevel (), 0, initTmpPos ()) {
+  : Attachment (mbox, initTmpLevel (), 0, initTmpPos ()),
+    emptyEMail (false),
+    previousKazAttachment (false) {
   DEF_LOG ("MainAttachment::MainAttachment", "");
   string line;
   for (; getline (mbox, line); )
@@ -377,6 +439,7 @@ MainAttachment::markSignificant (const streamoff &minAttachSize, ifstream &mbox)
   DEF_LOG ("MainAttachment::markSignificant", "minAttachSize: " << minAttachSize);
   bool plainMarked (false), htmlMarked (false);
   markDisclaim (plainMarked, htmlMarked);
+  emptyEMail = ! (plainMarked || htmlMarked);
   Attachment::markSignificant ("", minAttachSize, mbox, allMarkedPtrs);
 }
 
@@ -420,7 +483,7 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
   int attachCount (0);
   string dirName, mediaName;
   for (Attachment *attachP : allMarkedPtrs) {
-    if (!attachP->toExtract)
+    if (attachP->isKazAttachment || !attachP->toExtract)
       continue;
     newPjEntry (attachCount, attachP->getContentType (), attachP->getAttachName (), dirName, mediaName);
     ++attachCount;
@@ -495,15 +558,15 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
 
 // ================================================================================
 void
-MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize) {
+MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, const AttachMode &attachMode) {
-  DEF_LOG ("MainAttachment::substitute", "minSize: " << minSize);
+  DEF_LOG ("MainAttachment::substitute", "minSize: " << minSize << " AttachMode: " << attachMode);
 
   // preparation
   extractPreviousKAZ (mbox);
   removePreviousArchive ();
   map<const string, const string> translateHtml;
   for (Attachment *attachP : allMarkedPtrs)
-    if (attachP->toExtract) {
+    if (attachP->toExtract && !attachP->isKazAttachment) {
       readDownloadUrl (attachP->downloadUrl);
       if (attachP->downloadUrl.empty ()) {
 	LOG ("no change");
@@ -512,7 +575,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
       }
       if (attachP->cid.length ()) {
 	string tmp (attachP->downloadUrl);
-	replaceAll (tmp, "&", "&amp;");
+	//replaceAll (tmp, "&", "&amp;");
       	translateHtml.insert (pair<const string, const string> (CID+attachP->cid, tmp));
       }
     }
@@ -521,19 +584,36 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
       continue;
     for (EmbeddedData &embedded : attachP->embeddedData)
       readDownloadUrl (embedded.downloadUrl);
+    // XXX test empty ?
   }
   readArchiveUrl ();
+  removePreviousArchive ();
   string plainDisclaim, htmlDisclaim;
   getDisclaim (plainDisclaim, htmlDisclaim);
   // copy email
   streamoff curPos = 0;
+  copy (mbox, outbox, curPos, contentPos);
+  curPos = contentPos;
+
+  if (plainDisclaim.size ()) {
+    if (emptyEMail && (attachMode & FOOTER)) {
+      // check no main text
+      LOG ("Force main text");
+      LOG_BUG (boundary.empty () || ! subAttachements.size (), /**/, "eMailShrinker: can't force add footer M9: : " << *this);
+      string content (plainDisclaim);
+      base64Encode (content);
+      outbox << boundary.substr (0, boundary.length () -2) << endl
+	     << KAZ_EMPTY_TEXT_PLAIN << endl
+	     << content << endl;
+      outbox.flush ();
+    }
+  }
   for (Attachment *attachP : allMarkedPtrs) {
     copy (mbox, outbox, curPos, attachP->beginInParent);
-
     LOG_BUG (attachP->toUpdate && attachP->toExtract, /**/, "eMailShrinker: bug M5: update and extract. pos: " << attachP->beginPos);
 
-    if (attachP->toExtract) {
+    if (attachP->toExtract || attachP->isKazAttachment) {
-      LOG ("skip Extracted");
+      LOG ("skip Extracted or previous attachments");
 
     } else if (attachP->toUpdate) {
       string textProp = attachP->getProp (contentTypeToken, textRegEx);
@@ -581,7 +661,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
       }
       if (isPlain)
 	removeSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
-      if (isDisclaimer) {
+      if (isDisclaimer && (attachMode & FOOTER)) {
 	if (isHtml) {
 	  for (string endTag : {BODY_END, HTML_END}) {
 	    LOG ("try tag: " << endTag);
@@ -604,6 +684,20 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
     outbox.flush ();
     curPos = attachP->endPos;
   }
+  if (plainDisclaim.size () && (attachMode & ATTACHMENT)) {
+    // XXX si pas de multipart
+    LOG ("Add kaz attachment");
+    LOG_BUG (boundary.empty () || ! subAttachements.size (), /**/, "eMailShrinker: can't add Kaz attachment M10: : " << *this);
+    streamoff lastPos = subAttachements.back ().endPos;
+    copy (mbox, outbox, curPos, lastPos);
+    curPos = lastPos;
+    string content (KAZ_HTML_CONTENT+htmlDisclaim+BODY_END+HTML_END);
+    base64Encode (content);
+    outbox << boundary.substr (0, boundary.length () -2) << endl
+	   << KAZ_ATTACHMENT_TEXT_HTML << endl
+	   << content << endl;
+    outbox.flush ();
+  }
   copy (mbox, outbox, curPos, endPos);
   outbox.close ();
 }

src/cpp/eMailShrinker.cpp

@@ -32,7 +32,7 @@
 // knowledge of the CeCILL-B license and that you accept its terms.	  //
 ////////////////////////////////////////////////////////////////////////////
 
-#define LAST_VERSION "2.0 2022-02-08 eMailShrinker"
+#define LAST_VERSION "2.3 2022-11-25 eMailShrinker"
 
 #include <iostream>
 #include <fstream>
@@ -66,8 +66,8 @@ usage (const string &msg = "", const bool &hidden = false) {
   cout << endl
        << "Usage: " << endl
        << "       A) " << prog << " -u mbox > url-list" << endl
-       << "       B) " << prog << " [-s size] [-d dirName}] mbox > file-list" << endl
+       << "       B) " << prog << " [-s size] [-d dirName] mbox > file-list" << endl
-       << "       C) " << prog << " [-s size] mbox altered-mbox < url-list" << endl
+       << "       C) " << prog << " [-s size] [-m {Footer|Attachment|Both}] mbox altered-mbox < url-list" << endl
        << endl << "  filter attachments" << endl << endl
        << "  A: list previous embded url need to be refresh (no added option)" << endl
        << "     => downloadURL list" << endl
@@ -108,15 +108,16 @@ main (int argc, char** argv) {
   DEF_LOG ("main:", "");
   prog = argv [0];
   bool
-    debugFlag (false),
     helpFlag (false),
     versionFlag (false),
     updateListFlag (false),
     useTheForceLuke (false),
-    listFlag (false);
+    listFlag (false),
+    debugFlag (false);
   string inputName, outputName;
   bfs::path extractDir (bfs::temp_directory_path ());
   SizeArg minAttachSize ("48 Ki");
+  AttachMode attachMode (FOOTER);
 
   try {
     mainDescription.add_options ()
@@ -125,6 +126,7 @@ main (int argc, char** argv) {
       ("size,s", value<SizeArg> (&minAttachSize)->default_value (minAttachSize), "minimum size for extration")
       ("updateList,u", bool_switch (&updateListFlag), "list URL need refresh")
       ("extractDir,d", value<bfs::path> (&extractDir)->default_value (extractDir), "set tmp directory name for extraction")
+      ("mode,m", boost::program_options::value<AttachMode> (&attachMode)->default_value (attachMode), "set attachment mode")
       ;
 
     hide.add_options ()
@@ -188,10 +190,10 @@ main (int argc, char** argv) {
   MainAttachment attachment (mbox);
   mbox.close ();
 
-  if (attachment.getBoundary ().empty ()) {
+  // if (attachment.getBoundary ().empty ()) {
-    cerr << "no attachment" << endl;
+  //   cerr << "no attachment" << endl;
-    return 1;
+  //   return 1;
-  }
+  // }
   // parse structure
   mbox.open (inputName);
   attachment.markSignificant (minAttachSize, mbox);
@@ -217,13 +219,17 @@ main (int argc, char** argv) {
     showTime ("Extraction");
     return 0;
   }
-
+  
   // case substitute
+  if (attachMode == NONE) {
+    cerr << endl << prog << ": attachMode can't be NONE (forced FOOTER mode)" << endl;
+    attachMode = FOOTER;
+  }
   mbox.open (inputName);
   ofstream outbox (outputName);
-  attachment.substitute (mbox, outbox, minAttachSize);
+  attachment.substitute (mbox, outbox, minAttachSize, attachMode);
   showTime ("Substitution");
   return 0;
 }
 
-// ================================================================================
+  // ================================================================================

src/cpp/jirafeauAPI.cpp

@@ -32,7 +32,7 @@
 // knowledge of the CeCILL-B license and that you accept its terms.	  //
 ////////////////////////////////////////////////////////////////////////////
 
-#define LAST_VERSION "1.0 2021-02-21 jirafeauAPI"
+#define LAST_VERSION "1.1 2022-10-30 jirafeauAPI"
 
 #include <iostream>
 #include <string>
@@ -281,12 +281,13 @@ main (int argc, char** argv) {
   }
 
   curl_easy_setopt (easyhandle, CURLOPT_MIMEPOST, multipart);
-  curl_easy_perform (easyhandle);
+  CURLcode res (curl_easy_perform (easyhandle));
   curl_easy_cleanup (easyhandle);
   cout << readBuffer << endl;
 
   showTime ("Upload");
-
+  if (res != CURLE_OK)
+    cerr << prog << " failed: " << curl_easy_strerror (res) << endl;
   return 0;
 }
 

src/cpp/kazMisc.cpp

@@ -50,7 +50,7 @@ using namespace kaz;
 
 static const string::size_type MAX_QUOTED_PRINTABLE_SIZE (78);
 
-const char *const kaz::base64Chars =
+const char *const kaz::base64Chars = 
   "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
   "abcdefghijklmnopqrstuvwxyz"
   "0123456789"

src/include/Attachment.hpp

@@ -35,9 +35,9 @@
 #ifndef _kaz_Attachment_hpp
 #define _kaz_Attachment_hpp
 
-#include <string>
-#include <regex>
 #include <map>
+#include <regex>
+#include <string>
 #include <utility>
 
 #include "EmbeddedData.hpp"
@@ -54,7 +54,7 @@ namespace kaz {
     //static const vector<const string> stringsToUpdate;
     static vector<string> stringsToUpdate;
     /*! mime tokens */
-    static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, RELATED, ALTERNATIVE;
+    static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, RELATED, ALTERNATIVE, KAZ_ATTACH_NAME;
     /*! pattern to extract mime values */
     static const regex nameRegEx, nameCharsetRegEx, boundaryRegEx, cidDefRegEx, textRegEx, multiRegEx;
 
@@ -100,7 +100,7 @@ namespace kaz {
     /*! char position of attachment content */
     streamoff contentPos, endPos;
     /*! properties of the attachment */
-    bool toExtract, toUpdate, toDisclaim;
+    bool toExtract, toUpdate, toDisclaim, isKazAttachment;
     /*! id of an image embedded in mbox */
     string cid;
     /*! url to replace the attachment */

src/include/MainAttachment.hpp

@@ -44,6 +44,13 @@ namespace kaz {
   namespace bfs = boost::filesystem;
 
   // ================================================================================
+  /*! place to add download link (footer, attachment or both) */
+  enum AttachMode { NONE = 0, FOOTER = 1, ATTACHMENT = 2, BOTH = (FOOTER|ATTACHMENT) };
+  extern const string attachModeLabels[];
+  extern const map<string, AttachMode> attachModeMap;
+  ostream &operator << (ostream &out, const AttachMode &attachMode);
+  istream &operator >> (istream &in, AttachMode &attachMode);
+
   /*! root level of e-mail structure */
   class MainAttachment : public Attachment {
   public:
@@ -83,6 +90,10 @@ namespace kaz {
     bfs::path	extractDir;
     /*! URL for download archives */
     string	archiveDownloadURL;
+    /*! if no main text in email can be used to add disclaim */
+    bool emptyEMail;
+    /*! if contain previous kaz attachment */
+    bool previousKazAttachment;
     
     /*! subset in the tree of all attachments to be consider for extraction or modification */
     vector<Attachment *>		allMarkedPtrs;
@@ -113,7 +124,7 @@ namespace kaz {
     /*! extract big attachments in mbox to extractDir and write to stdout le dirname of each extraction */
     void extract (ifstream &mbox, const SizeArg &minSize) const;
     /*! substitute big attachments by the url give in stdin */
-    void substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize);
+    void substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, const AttachMode &attachMode);
   };
 
   // ================================================================================

src/mainpage.md

@@ -0,0 +1,10 @@
+**Dépollueur**
+
+Main Programmes:
+  * [eMailShrinker](eMailShrinker_8cpp.html)
+  * [jirafeauAPI](jirafeauAPI_8cpp.html)
+
+Main classes:
+  * [MainAttachment](classkaz_1_1MainAttachment.html)
+  * [Attachment](classkaz_1_1Attachment.html)
+  

test/mails/expectedResults.sh

@@ -0,0 +1,32 @@
+#/bin/bash
+
+# GME
+# Ici on défini une liste de tests à effectuer sur les mails reçus (et donc dépollués !).
+# Touts les mails sont concatennés avant vérification (egrep) des regexp donc on veillera, 
+# donc nommer différement les pièces sur les mails de tests si on a besoin de retrouver quel est le mail qui plante
+
+# pour chaque test
+TESTS_DESCRIPTION=(); # Une description pour l'affichage du résultat
+TESTS_REGEXP=(); # Quoi qu'on doit trouver ?
+TESTS_MATCHING_LINES=(); # Combien d'occurences de ça dans l'ensemble des messages ?
+
+
+TESTS_DESCRIPTION+=("Le nombre de mails correctement dépollués");
+TESTS_REGEXP+=(".*Attention : Kaz a \S+ ce message.*");
+TESTS_MATCHING_LINES+=(1);
+
+TESTS_DESCRIPTION+=("La piece est correctement nommée pour un mail normal");
+TESTS_REGEXP+=("\\* \"pdftest.pdf\" <https://depot.[^/]*/");
+TESTS_MATCHING_LINES+=(1);
+
+TESTS_DESCRIPTION+=("La piece est correctement nommée pour un mail sans texte");
+TESTS_REGEXP+=("\\* \"pdftestmailsanstexte.pdf\" <https://depot.[^/]*/");
+TESTS_MATCHING_LINES+=(1);
+
+TESTS_DESCRIPTION+=("La piece est correctement nommée pour une piece sans guillemets");
+TESTS_REGEXP+=("\\* \"pdftestSansGuillemets.pdf\" <https://depot.[^/]*/");
+TESTS_MATCHING_LINES+=(1);
+
+TESTS_DESCRIPTION+=("L'encodage du mail est correct");
+TESTS_REGEXP+=(".*é.*");
+TESTS_MATCHING_LINES+=(0);

test/runTests.sh

@@ -0,0 +1,456 @@
+#/bin/bash
+
+# --- GME ---
+# Envois full automatisés de mails dits bizarres vers un serveur kaz, 
+# dans le but de tester bon fonctionnement du dépollueur
+#
+# On enregistre en .eml dans le dossier mails, des messages "mal construits" pouvant causer pb au dépollueur.
+# Les mails ont les champs suivants à l'interieur: %DATE% %FROM_ADDRESS% %TO_ADDRESS%
+# Le fichier expectedResults permet de définir des regexp qu'on doit vérifier sur le mail reçu.
+#
+# Le script les envoie un par un via l'utilitaire swaks
+#    - interne -> interne
+#    - interne -> externe
+#    - externe -> interne
+#
+# Un petit warning, ce script fait du mailing moche (on balance un paquet de mails volontairement mal construits).
+# Vous savez ce que vous faites ! Et si vous ne savez pas, ne le faites pas.
+# En particulier si on utilise des mails qui sortent et entrent ... (blacklistings / etc ...)
+# On préfèrera faire les tests sur un environnement de dev bien sûr !
+
+TEST_DIR=$(cd "$(dirname $0)"; pwd)
+. $TEST_DIR/mails/expectedResults.sh
+NBMAILS=$(ls $TEST_DIR/mails/*.eml | wc -l)
+
+# random key
+KEY="TESTDEP-$(echo $RANDOM | md5sum | head -c 20;)"
+
+# ou qu'il est le depollueur ?
+SERVEUR_TO_TEST=
+SERVEUR_IS_LOCALHOST=0
+
+# s'il est pas en local, il faut bien pouvoir s'identifier 
+USERFROM=
+USERTO=
+PASSWORD=
+
+# un mail externe pour tester entrant / sortant
+EXTERNAL_EMAIL=
+EXTERNAL_SMTP=
+EXTERNAL_USER=
+EXTERNAL_PASSWORD=
+EXTERNAL_SMTP_AUTH=1
+EXTERNAL_TESTS_ENABLED=1
+EXTERNAL_SMTP_ENABLED=1
+EXTERNAL_ONLY=0
+
+#un peu de debug
+VERBOSE=0
+
+#ou si on préfère ne rien savoir
+QUIET=0
+
+#swaks est installé ? 
+SWAKS_OK=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
+
+# les résultats
+SKIP_RESULTS=0
+KEEP_MAILS_ON_SERVEURS=0
+
+#mutt est installé ? 
+MUTT_OK=$(dpkg-query -W --showformat='${Status}\n' mutt 2>/dev/null | grep "install ok installed")
+
+
+
+usage () {
+	echo "${PRG} [-h] -s <smtp_a_tester> -a <user> [-l] [-p <password>] [-v|-q] [-eq] [-ea] [-em <mail-ext>] [-es <smtp-ext>] [-eu <user-ext>] [-ep <password-ext>]"
+	echo "    Tests full-automatisés du dépollueur."
+    echo "    On envoie une série de mails et on vérifie que le lien généré est bon."
+	echo "    -h | --help           : Display this help."
+	echo "    -s | --server         : Quel serveur je teste ? (fqnd)"
+	echo "    -l | --localhost      : si le script est execute depuis le serveur test"
+	echo "    -f | --from           : Un compte à utiliser sur ce serveur en tant qu'expediteur"
+	echo "    -t | --to             : Un compte à utiliser sur ce serveur en tant que destinataire (optionnel, l expediteur est utilise sinon)"
+	echo "    -p | --password       : Le mot de passe de l'expéditeur si le script est executé à distance"
+	echo "    -ed | --ext-disabled  : Ne pas faire de tests externes"
+    echo "    -eo | --ext-only      : Ne faire que les tests externes"
+	echo "    -em | --ext-mail      : Un mail externe pour des tests vers / depuis l'exterieur"
+	echo "    -es | --ext-server    : Quel serveur je teste ?"
+	echo "    -eu | --ext-user      : Un identifiant pour smtp ext si différents de mail"
+	echo "    -ep | --ext-password  : Le mot de passe associé pour le ext-user ou mail"
+	echo "    -ena | --ext-no-auth  : Le smtp externe n'a pas besoin d'authentifications (FAI par exemple)"
+    echo "    -rd | --results-disabled : Ne pas tenter d'aller chercher les résultats"
+    echo "    -ksm | --keep-sended-mails : Laisser les messages envoyés sur les serveurs de destination"
+	echo "    -v or -V display more informations"
+	echo "    -q or -Q Quiet mode"
+    echo "Un petit warning, ce script fait du mailing moche (on balance un paquet de mails volontairement mal construits). "
+    echo "Vous savez ce que vous faites ! Et si vous ne savez pas, ne le faites pas."
+}
+
+cleanupPackage () {
+    # on vire swaks si on l'a installé
+    # eventuellement on le fait même silently
+    package=$1;
+    quietMode=$2;
+    [ -z $quietMode ] && quietMode=0;
+    [ $quietMode -eq 0 ] && echo "$package a été installé, vous permettez que je désinstalle .. !";
+    if [ $quietMode -eq 0 ] 
+    then
+        apt-get -y remove $package;
+        apt-get -y purge $package;
+    else
+        apt-get -yq remove $package;
+        apt-get -yq purge $package;
+    fi;
+}
+
+# récupération des arguments
+[ -z "$1" ] && { usage;exit; }
+for arg in $*
+do
+    case "$arg" in
+        '-h' | '--help' )
+        usage
+        shift
+        exit;;
+        '-s' | '--server')
+        shift;
+        SERVEUR_TO_TEST=$1
+        shift;;
+        '-l' | '--localhost')
+        SERVEUR_IS_LOCALHOST=1
+        shift;;
+        '-f' | '--from')
+        shift;
+        USERFROM=$1
+        shift;;
+        '-p' | '--password')
+        shift;
+        PASSWORD=$1
+        shift;;
+        '-t' | '--to')
+        shift;
+        USERTO=$1
+        shift;;
+        '-ed' | '--ext-disabled')
+        EXTERNAL_TESTS_ENABLED=0;
+        EXTERNAL_SMTP_ENABLED=0;
+        shift;;
+        '-em' | '--ext-mail')
+        shift;
+        EXTERNAL_EMAIL=$1
+        shift;;
+        '-es' | '--ext-server')
+        shift;
+        EXTERNAL_SMTP=$1
+        shift;;
+        '-eu' | '--ext-user')
+        shift;
+        EXTERNAL_USER=$1
+        shift;;
+        '-ep' | '--ext-password')
+        shift;
+        EXTERNAL_PASSWORD=$1
+        shift;;
+        '-ena' | '--ext-no-auth')
+        EXTERNAL_SMTP_AUTH=0
+        shift;;
+        '-eo' | '--ext-only')
+        EXTERNAL_ONLY=1
+        shift;;
+        '-v' | '-V' )
+        VERBOSE=1
+        shift;;
+        '-q' | '-Q' )
+        QUIET=1
+        shift;;
+        '-rd' | '--results-disabled')
+        SKIP_RESULTS=1
+        shift;;
+        '-ksm' | '--keep-sended-mails')
+        KEEP_MAILS_ON_SERVEURS=1
+        shift;;
+        *)
+    esac
+done;
+
+[ $VERBOSE -eq 1 -a $QUIET -eq 1 ] && echo 'Les options -q et -v ne sont pas compatibles.' && usage;
+
+[ $VERBOSE -eq 1 ] && echo "Checking for swaks : $SWAKS_OK";
+if [ -z "$SWAKS_OK" ]; 
+then
+    if [ $QUIET -eq 0 ]
+    then
+        echo "Swaks doit être installé, je le fait pour vous si j'ai les droits !";
+        while true; do
+            read -p "On y va ? [Yn] :" yn
+            case $yn in
+                [YyoO]* ) break;;
+                [Nn]* ) echo "Swaks doit être installé, les tests en dépendent"; exit;;
+                * ) break;;
+            esac
+        done;
+        apt-get update;
+        apt-get -y install swaks;
+    else
+        apt-get -q update;
+        apt-get -qy install swaks;
+    fi;
+
+    SWAKS_INSTALLED=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
+    if [ -z "$SWAKS_INSTALLED" ] 
+    then
+        echo "Swaks doit être installé, et je ne suis pas root visiblement.";
+        echo "Soit le faire à la main, soit je m'en charge (apt install et remove à la fin), mais il me faut les droits root !";
+        exit;
+    fi;
+
+    trap 'cleanupPackage swaks $QUIET' EXIT;
+fi
+
+if [ $SKIP_RESULTS -eq 0 -a -z "$MUTT_OK" ]; 
+then
+    [ $VERBOSE -eq 1 ] && echo "Checking for mutt : $MUTT_OK";
+    if [ $QUIET -eq 0 ]
+    then
+        echo "Mutt doit être installé, je le fait pour vous si j'ai les droits !";
+        while true; do
+            read -p "On y va ? [Yn] :" yn
+            case $yn in
+                [YyoO]* ) break;;
+                [Nn]* ) echo "Mutt doit être installé, les tests en dépendent"; exit;;
+                * ) break;;
+            esac
+        done;
+        apt-get update;
+        apt-get -y install mutt;
+    else
+        apt-get -q update;
+        apt-get -qy install mutt;
+    fi;
+
+    MUTT_INSTALLED=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
+    if [ -z "$MUTT_INSTALLED" ] 
+    then
+        echo "Mutt doit être installé, et je ne suis pas root visiblement.";
+        echo "Soit le faire à la main, soit je m'en charge (apt install et remove à la fin), mais il me faut les droits root !";
+        exit;
+    fi;
+
+    trap 'cleanupPackage mutt $QUIET' EXIT;
+fi
+
+# si les paramètres n'ont pas été renseignés via les arguments du script, 
+# on prompt ceux qui sont necessaires
+
+[ $QUIET -eq 1 -a -z "$SERVEUR_TO_TEST" ] && SERVEUR_TO_TEST="mail.kaz.local"; #en mode quiet, on prend les options par défaut
+if [ -z "$SERVEUR_TO_TEST" ]
+    then
+        [ $VERBOSE -eq 1 ] && echo "-- Le serveur n'a pas été renseigné (il faut un fqdn):";
+        read -p "Le serveur à tester ? [mail.kaz.local] :" SERVEUR_TO_TEST;
+        [ -z "$SERVEUR_TO_TEST" ] && SERVEUR_TO_TEST="mail.kaz.local";
+fi;
+
+[ $QUIET -eq 1 -a -z "$USERFROM" ] && SERVEUR_TO_TEST="contact1@kaz.local"; #en mode quiet, on prend les options par défaut
+if [ -z "$USERFROM" ]
+    then
+        [ $VERBOSE -eq 1 ] && echo "-- Il me faut également un compte mail qui envoie :";
+        read -p "Email Expediteur ? [contact1@kaz.local]" USERFROM;
+        [ -z "$USERFROM" ] && USERFROM="contact1@kaz.local";
+fi;
+[ -z "$USERTO" ] && USERTO="$USERFROM"; #sauf si précisé en option
+
+# si on est en local, normalement les mails partent sans besoin de renseigner l'authentification
+# sinon il faut un mot de passe pour le smtp
+
+if { [ $SKIP_RESULTS -eq 0 ] || [ $SERVEUR_IS_LOCALHOST -eq 0 ]; } && [ -z "$PASSWORD" ];
+    then
+        [ $VERBOSE -eq 1 ] && echo "-- Le password à utiliser n'a pas été renseigné :";
+        read -s -p "Le password ? [toto]" PASSWORD;
+        [ -z "$PASSWORD" ] && PASSWORD="toto";
+fi;
+
+# utilisation d'un mail exterieur et eventuellement un smtp exterieur
+if [ -z "$EXTERNAL_EMAIL" -a $EXTERNAL_TESTS_ENABLED -eq 1 -a $QUIET -eq 0 ]
+then
+    [ $VERBOSE -eq 1 ] && echo "-- Les tests vers/depuis l'exterieur n'ont pas été desactivé via l'option -ed.";
+    while true; do
+        read -p "On fait des tests depuis / vers l'exterieur aussi ? [Yn] :" yn
+        case $yn in
+            [YyoO]* ) break;;
+            [Nn]* ) EXTERNAL_TESTS_ENABLED=0; EXTERNAL_SMTP_ENABLED=0; break;;
+            * ) break;;
+        esac
+    done;
+    if [ $EXTERNAL_TESTS_ENABLED -eq 0 ] 
+    then
+        [ $VERBOSE -eq 1 ] && echo "-- Les tests vers/depuis l'exterieur annulés";
+    else
+        [ $VERBOSE -eq 1 ] && echo "-- Ok mais je n'ai toujours pas l'adresse exterieure à utiliser.";
+        read -p "Mail externe ? :" EXTERNAL_EMAIL;
+    fi
+fi
+if [ -z "$EXTERNAL_SMTP" -a $EXTERNAL_TESTS_ENABLED -eq 1 -a $QUIET -eq 0 ]
+then
+    [ $VERBOSE -eq 1 ] && echo "-- smtp, je pourrais le deviner à partir du mail mais ... j'ai la flemme là."; 
+    echo "Pour les tests Depuis l'exterieur, il me faut aussi un smtp.Laisser vide si pas de tests depuis l'exterieur."
+    read -p "smtp externe ? :" EXTERNAL_SMTP;
+    [ -z $EXTERNAL_SMTP ] && EXTERNAL_SMTP_ENABLED=0;
+fi
+# authent pour le smtp ext
+if [ -z "$EXTERNAL_USER" -a $EXTERNAL_SMTP_AUTH -eq 1 -a $EXTERNAL_SMTP_ENABLED -eq 1 -a $QUIET -eq 0 ]
+then
+    [ $VERBOSE -eq 1 ] && echo "L'authentification au smtp externe n'a pas été désactivée avec l'option -ea :";
+    [ $VERBOSE -eq 1 ] && echo "Le user à utiliser pour ce smtp externe n'a pas été renseigné :";
+    read -p "Le user du smtp ? " EXTERNAL_USER;
+fi
+if [ -z "$EXTERNAL_PASSWORD" -a $EXTERNAL_SMTP_AUTH -eq 1 -a $EXTERNAL_SMTP_ENABLED -eq 1 -a $QUIET -eq 0 ]
+then
+    [ $VERBOSE -eq 1 ] && echo "L'authentification au smtp externe n'a pas été désactivée avec l'option -ea :";
+    [ $VERBOSE -eq 1 ] && echo "Le password à utiliser pour ce smtp externe n'a pas été renseigné :";
+    read -s -p "Le password pour le smtp ? " EXTERNAL_PASSWORD;
+fi
+
+####
+#### Fin de la récupération des paramètres du script. On balance les mails !
+####
+
+if [ $VERBOSE -eq 1 ] 
+then 
+    echo 'Paramètres : '
+    echo '------------'
+    echo " Serveur : $SERVEUR_TO_TEST"
+    echo " Expéditeur interne : $USERFROM"
+    echo " Destinataire interne : $USERTO"
+    echo ' Password : ***'
+    echo " Tests vers externes : $EXTERNAL_TESTS_ENABLED"
+    echo " Tests depuis externe : $EXTERNAL_SMTP_ENABLED"
+    echo " Mail externe : $EXTERNAL_EMAIL"
+    echo " Auth externe : $EXTERNAL_USER"
+    echo ''
+    echo ''
+fi
+if [ $QUIET -eq 0 ] 
+then 
+    sleep 2;
+    echo "OK on va y aller ... mais j'attend encore 3 secondes pour donner le temps pour un Ctrl C"
+    sleep 3;
+fi
+if [ $VERBOSE -eq 1 ] 
+then 
+    echo "C'est parti !"; 
+    # vous êtes sûr ? bon d'accord c'est chiant, 
+    # mais si on a cree des mails vraiment veroles, et qu'on est en train de les balancer sur la en prod, on peut avoir un doute ...
+fi
+sleep 2;
+
+if [ $EXTERNAL_ONLY -eq 0 ]
+then
+    echo "1 ----- $NBMAILS Mails envoyés depuis $USERFROM vers $USERTO ----";
+    for MAIL in "$TEST_DIR/mails/"*.eml
+    do
+        echo "$MAIL";
+        options=(-f $USERFROM -t $USERTO -d $MAIL --h-From: "\"$KEY\" <$USERFROM>");
+        [ $SERVEUR_IS_LOCALHOST -eq 0 ] &&  options+=(-s $SERVEUR_TO_TEST -p 587 --tlso -au $USERFROM -ap $PASSWORD); 
+        [ $QUIET -eq 1 ] &&  options+=(--silent);
+        [ $VERBOSE -eq 0 ] &&  options+=(-n);
+        swaks "${options[@]}";
+    done
+fi
+
+if [ $EXTERNAL_TESTS_ENABLED -eq 1 ]
+then
+    echo "2 ----- $NBMAILS Mails envoyés depuis $USERFROM vers $EXTERNAL_USER ----";
+    for MAIL in "$TEST_DIR/mails/"*.eml
+    do
+        options=(-f $USERFROM -t $EXTERNAL_EMAIL -d $MAIL -s $SERVEUR_TO_TEST --tlso --h-From: "\"$KEY\" <$USERFROM>");
+        [ $SERVEUR_IS_LOCALHOST -eq 0 ] &&  options+=(-p 587 -au $USERFROM -ap $PASSWORD); # on rajoute l'authetification
+        [ $QUIET -eq 1 ] &&  options+=(--silent);
+        [ $VERBOSE -eq 0 ] &&  options+=(-n);
+        swaks "${options[@]}";
+    done
+fi
+
+if [ $EXTERNAL_SMTP_ENABLED -eq 1 ]
+then
+    if [ $SERVEUR_IS_LOCALHOST -eq 1 ] 
+    then 
+        [ $QUIET -eq 0 ] && echo "3 ----- Mails envoyés depuis l'extérieur vers localhost ... heu une autre fois en fait ... ";
+        [ $VERBOSE -eq 1 ] && echo "-- Mais si vous renseignez un fqdn pour le serveur je pourrais essayer ...";
+    else
+        echo "3 ----- $NBMAILS Mails envoyés depuis $EXTERNAL_EMAIL vers $USERTO ----";
+        for MAIL in "$TEST_DIR/mails/"*.eml
+        do
+            options=(-f $EXTERNAL_EMAIL -t $USERTO -d $MAIL -s $EXTERNAL_SMTP -p 587 --tlso  --h-From: "\"$KEY\" <$USERFROM>");
+            [ $EXTERNAL_SMTP_AUTH -eq 1 ] && options+=(-au $EXTERNAL_USER -ap $EXTERNAL_PASSWORD);
+            [ $QUIET -eq 1 ] &&  options+=(--silent);
+            [ $VERBOSE -eq 0 ] &&  options+=(-n);
+            swaks "${options[@]}";
+        done
+    fi
+fi
+
+####
+#### Et bien maintenant on va voir les résultats !
+####
+
+if [ $SKIP_RESULTS -eq 0 ]
+then 
+    if [ $VERBOSE -eq 1 ] 
+    then 
+        echo "-- On va maintenant chercher les mails en imap pour checker que tout est bon !";
+        echo "-- Creation d'un repertoire temporaire /tmp/$KEY";
+        echo "-- Fichier de conf pour mutt /tmp/$KEY/muttrc";
+    fi   
+     
+    sleep 5;
+    [ $VERBOSE -eq 1 ] && echo "-- OK on va y aller ... mais j'attend encore 3 secondes pour donner le temps aux messages d'être traités par les serveurs ...";
+    sleep 3;
+
+    mkdir /tmp/$KEY 
+    echo "set imap_user=$USERFROM
+set ssl_force_tls = yes
+set ssl_starttls  = yes
+set imap_pass=$PASSWORD
+set folder=imaps://$SERVEUR_TO_TEST:993
+set spoolfile = +INBOX
+set pipe_decode=yes
+set wait_key=no
+mailboxes +INBOX
+" > /tmp/$KEY/muttrc
+
+    
+    [ $VERBOSE -eq 1 ] && echo "-- on télécharge les mails qu'on colle dans un txt";
+    mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><tag-pattern>all<enter><tag-prefix><pipe-message>cat >> /tmp/$KEY/mails.eml<enter><quit>'"
+    
+    [ $VERBOSE -eq 1 ] && echo "-- Et on fait le ménage sur le serveur";
+    if [ $KEEP_MAILS_ON_SERVEURS -eq 1 ]
+    then
+        [ $VERBOSE -eq 1 ] && echo "-- On les enregistre dans un sous dossier $KEY";
+        mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><tag-pattern>all<enter><tag-prefix><decode-save><enter><enter><quit>y'"
+    else
+        [ $VERBOSE -eq 1 ] && echo "-- On supprime les mails identifiés par la clef";
+        mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><delete-pattern>all<enter><enter><quit>y'"
+    fi
+    
+    
+    NBTESTS=${#TESTS_DESCRIPTION[@]};
+    RED='\033[0;31m';
+    GREEN='\033[0;32m';       # Green
+    NC='\033[0m';         # No Color
+    [ $VERBOSE -eq 1 ] && echo "-- On passe les $NBTESTS tests définis dans le repertoire de mails";
+
+    echo "--------  RESULTATS DES TESTS ------ ";
+    for (( i=0; i<${NBTESTS}; i++ ));
+    do
+        RESULTCOUNT=$(egrep -c "${TESTS_REGEXP[$i]}" /tmp/$KEY/mails.eml);
+        [ -z $RESULTCOUNT ] && RESULTCOUNT=0;
+        if [ $RESULTCOUNT -eq ${TESTS_MATCHING_LINES[$i]} ]
+        then
+            echo -e "--$i-- ${TESTS_DESCRIPTION[$i]} : ${GREEN}OK${NC}";
+        else
+            echo -e "--$i-- ${TESTS_DESCRIPTION[$i]} : ${RED}KO${NC}";
+        fi
+    done
+
+fi