38 Commits

Author SHA1 Message Date
fab
94659482cf Mise à jour de 'src/bash/filter.sh' 2023-10-23 12:22:39 +02:00
fab
3ac1bec5d9 Mise à jour de 'src/bash/filter.sh' 2023-10-21 02:07:06 +02:00
fab
4c61cf5c81 Mise à jour de 'src/bash/filter.sh' 2023-10-21 01:21:25 +02:00
fab
aea24bbe89 Mise à jour de 'src/bash/filter.sh' 2023-10-20 22:51:11 +02:00
bd3a2bc0c8 update var name 2023-10-20 19:29:20 +02:00
b0e3c5b390 test skip 2023-10-20 19:25:26 +02:00
eb5b3b3ec7 fix no space after token in MIME header 2023-09-17 18:36:25 +02:00
dae4379ad7 fix2 filterTest.sh 2023-09-17 17:20:08 +02:00
35a22de854 fix filterTest.sh parameters 2023-09-17 17:11:22 +02:00
3533111a69 update version 2023-04-23 18:13:04 +02:00
3a6b4b3a90 fix "first character of the attachment name was missing" 2023-04-23 17:56:03 +02:00
dce312c33e retry upload 2023-03-03 15:22:13 +01:00
f6db4af4fc upgrade Jirafeau 4.5 2023-01-28 08:29:40 +01:00
f51dc3f7a3 upgrade filterTest (exit code) 2023-01-23 14:09:08 +01:00
2980767d53 update version 2023-01-22 07:27:35 +01:00
bb737e00f7 fix Rainllop 2023-01-22 07:26:24 +01:00
cf7c4102af fix LOG in filter.sh 2023-01-21 19:35:25 +01:00
29ba4fd0d6 fix comma in filename (curl param) 2023-01-16 17:30:37 +01:00
d172596a5d same remove special char in filterTest.sh 2023-01-13 17:08:11 +01:00
33f539a3bb remove special char in password 2023-01-13 17:08:24 +01:00
fc0455e7a8 fix version 2023-01-01 00:08:54 +01:00
e3bbc5ddde fix base64 (==) 2023-01-01 00:07:36 +01:00
38085291ea fix base42 (2) 2022-12-28 15:16:42 +01:00
9141adfb5b fix base64 2022-12-28 15:03:41 +01:00
c9ab7d4f49 fix no content-type in source mbox 2022-12-27 14:48:12 +01:00
18d1214681 update attach filename 2022-12-27 12:01:04 +01:00
285792acfc fix curl filter / fix BOTH 2022-12-27 10:14:47 +01:00
53c5c29a14 fix filter log 2022-12-25 08:49:27 +01:00
61bdc3a4ba fix rfc2047 / filter log / filterTest options 2022-12-25 06:57:44 +01:00
596ae82fe4 fix PGP (skip signed message) 2022-12-23 11:39:44 +01:00
694570a454 update filterTest.sh 2022-12-21 19:54:44 +01:00
122788c2ac fix TAG KAZ_PLAIN_STOP 2022-12-19 18:01:08 +01:00
49a339bdd8 update version 2022-12-17 08:59:10 +01:00
fbaaf6ea3d fix unexpected boundary not in multipart 2022-12-17 08:47:22 +01:00
df7a25d331 double remove & / display version 2022-12-09 08:54:02 +01:00
8e74856ad2 update version 2022-12-08 16:55:12 +01:00
1540688c18 fix double mime, windows \r 2022-12-08 16:42:20 +01:00
51865cfce2 add MODE "none" 2022-12-01 07:47:25 +01:00
24 changed files with 761 additions and 94003 deletions

View File

@ -33,7 +33,7 @@ depollueur/
## Compilation
```bash
sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev doxygen
sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev doxygen dos2unix
git clone https://git.kaz.bzh/KAZ/depollueur.git
# or for contributors :

View File

@ -77,7 +77,7 @@ define ('M_WELCOME', "<p>Informations concernant le compte : <b>___SENDER___</b>
define ('M_INCONSISTENT_DATES',
" (dates incoh&eacute;antes avec ___FILENAME___ : ___DIRTIME___ != ___FILETIME___)");
define ('A_ACTION', 'a'); // action : T_LOGIN, T_LOGOUT, A_MODE(footer|attachment|both), A_RECORD+(on|off), A_PERIOD(minute|hour|day|week|month|quarter), A_LANG(fr|en|br)
define ('A_ACTION', 'a'); // action : T_LOGIN, T_LOGOUT, A_MODE(none|footer|attachment|both), A_RECORD+(on|off), A_PERIOD(minute|hour|day|week|month|quarter), A_LANG(fr|en|br)
define ('A_GET', 'g'); // get archive
define ('A_HASH', 'h'); // file to update or delete
define ('A_OPEN_TOKEN', 'o'); // ask token
@ -114,7 +114,7 @@ define ('T_ARCHIVE_TITLE', "archive_content");
define ('T_ARCHIVE_MIME', "text/kaz_email_archive");
$modeText = ['footer' => "pied de page", 'attachment' => "pi&egrave;ce jointe", 'both' => "les deux"];
$modeText = ['none' => "sans", 'footer' => "pied de page", 'attachment' => "pi&egrave;ce jointe", 'both' => "les deux"];
$trackText = ['on' => "oui", 'off' => "non"];
$periodText = ['minute' => "minute", 'hour' => "heure", 'day' => "jour", 'week' => "semaine", 'month' => "mois"];
// XXX , 'quarter' => "trimestre"];

View File

@ -89,6 +89,8 @@ if (!empty($delete_code) && $delete_code == $link['link_code']) {
'</p></div>';
} else { ?>
<div>
<form action="<?php echo 'f.php?h=' . $link_name . '&amp;d=' . $delete_code; ?>" method="post" id="submit_delete_post" class="form login">
<input type="hidden" name="do_delete" value="1" />
<form action="f.php" method="post" id="submit_delete_post" class="form login">
<input type="hidden" name="do_delete" value=1/>
<fieldset>
@ -102,8 +104,6 @@ if (!empty($delete_code) && $delete_code == $link['link_code']) {
</td></tr>
<tr><td>
<input type="submit" id="submit_delete" value="<?php echo t('DELETE'); ?>"
<?php $action_delete="'f.php?h=' . $link_name . '&amp;d=' . $delete_code'; document.getElementById('submit_delete').submit ();"; ?>
onclick="document.getElementById('submit_delete_post').action='<?php echo $action_delete; ?>" />
</td></tr>
</table>
</fieldset></form></div><?php
@ -145,7 +145,7 @@ if (!empty($link['key'])) {
'<legend>' . t('PSW_PROTEC') .
'</legend><table><tr><td>' .
t('GIMME_PSW') . ' : ' .
'<input type = "password" name = "key" />' .
'<input type = "password" name = "key" autocomplete = "current-password"/>' .
'</td></tr>' .
'<tr><td>' .
t('USING_SERVICE'). ' <a href="tos.php" target="_blank" rel="noopener noreferrer">' . t('TOS') . '</a>.' .
@ -276,12 +276,17 @@ elseif ($link['crypted']) {
}
/* Read file. */
else {
if ($cfg['use_xsendfile']) {
$file_web_path = preg_replace('#^' . $_SERVER['DOCUMENT_ROOT'] . '#', '', VAR_FILES);
header('X-Sendfile: ' . $file_web_path . $p . $link['hash']);
} else {
$r = fopen(VAR_FILES . $p . $link['hash'], 'r');
while (!feof($r)) {
print fread($r, 1024);
}
fclose($r);
}
}
if ($link['onetime'] == 'O') {
jirafeau_delete_link($link_name);

View File

@ -1,121 +0,0 @@
<?php
define ('JIRAFEAU_ROOT', dirname (__FILE__) . '/');
require (JIRAFEAU_ROOT . 'lib/settings.php');
require (JIRAFEAU_ROOT . 'lib/functions.php');
require (JIRAFEAU_ROOT . 'lib/lang.php');
@set_time_limit (0);
/* Remove errors. */
@error_reporting (0);
if (isset ($_REQUEST ['l']) && !empty ($_REQUEST ['l']))
$linksPass = explode ("/", $_REQUEST ["l"]);
else if (isset ($_REQUEST ['h']) && !empty ($_REQUEST ['h']))
$linksPass = $_REQUEST ["h"];
else
die ("no links");
if (!is_array ($linksPass))
die ("no list is given: ".$h);
$notFoundCount=0;
$map = [];
// First pass: check
foreach ($linksPass as $line) {
if (strpos ($line, '~') !== false)
$couple = explode ("~", $line, 2);
else
$couple = explode ("/", $line, 2);
if (count ($couple) == 0)
continue;
$link_name = $couple [0];
if (!$link_name)
continue;
$crypt_key = count ($couple) == 2 ? $couple [1] : "";
if (!preg_match ('/[0-9a-zA-Z_-]+$/', $link_name))
die ("bad link format : ".$link_name);
$link = jirafeau_get_link ($link_name);
if (count ($link) == 0) {
++$notFoundCount;
continue;
}
$key = $link['key'];
if ($key) {
preg_match ( '/[0-9a-zA-Z_-]+/', $link['key'], $matches);
$key = $matches[1];
}
if ($key && (empty ($crypt_key) || $key != $crypt_key))
die ("bad key for ".$link);
$map [$link_name] = $crypt_key;
}
// second pass: send
if (isset ($_REQUEST ['n']) && !empty ($_REQUEST ['n']))
$dirname=$_REQUEST ['n'];
else
$dirname="kaz-".date ("Ymd-His");
$tmpFileName = tempnam (sys_get_temp_dir (), $dirname."-");
$zip = new ZipArchive;
if (!$zip)
die ("can't create tmp");
if ($zip->open ($tmpFileName.".zip", ZipArchive::CREATE) !== TRUE)
die ("can't create tmp");
if ($notFoundCount) {
$zip->addFromString ($dirname."-Avertissement.txt", $notFoundCount. ($notFoundCount ? " fichier est expiré." : " fichiers sont expirés."));
}
$single_name=[];
foreach ($map as $link_name => $crypt_key) {
$link = jirafeau_get_link ($link_name);
$p = s2p ($link ['hash']);
$src_name = $dst_name = $link['file_name'];
if (in_array ($src_name, $single_name))
for ($i = 0; $i < 10000; ++$i) {
$dst_name = sprintf ("%s-%2d", $src_name, $i);
if (!in_array ($dst_name, $single_name))
break;
}
$single_name[]=$dst_name;
// send
if ($link['crypted']) {
$m = mcrypt_module_open ('rijndael-256', '', 'ofb', '');
$md5_key = md5 ($crypt_key);
$iv = jirafeau_crypt_create_iv ($md5_key, mcrypt_enc_get_iv_size ($m));
mcrypt_generic_init ($m, $md5_key, $iv);
$r = fopen (VAR_FILES . $p . $link['hash'], 'r');
$content = "";
while (!feof ($r)) {
$dec = mdecrypt_generic ($m, fread ($r, 1024));
$content .= $dec;
ob_flush ();
}
fclose ($r);
$zip->addFromString ($dirname."/".$dst_name, $content);
mcrypt_generic_deinit ($m);
mcrypt_module_close ($m);
continue;
}
$zip->addFile (VAR_FILES . $p . $link['hash'], $dirname."/".$dst_name);
}
$zip->close ();
if (!is_file ($tmpFileName.".zip"))
die ("can't retreive tmp");
header ("HTTP/1.0 200 OK");
header ("Content-Type: application/zip");
header ('Content-Disposition: filename="'.$dirname.'.zip"');
$r = fopen($tmpFileName.".zip", 'r');
while (!feof ($r)) {
print fread ($r, 1024);
ob_flush ();
}
fclose ($r);
unlink ($tmpFileName.".zip");
unlink ($tmpFileName);

181
src/bash/filter.sh Normal file → Executable file
View File

@ -35,10 +35,13 @@
##########################################################################
# - installer l'utilitaire apg pour génération de mot de passes
# - installer l'utilitaire dos2unix
# - le contenu de INSPECT_DIR doit être accessible en écriture pour le
# proriétaire du script
# - shrinkEMail et jirafeau.sh doivent être accessible en execution pour
# le roriétaire du script
# - il faut que root fasse avant :
# mkdir -p "${DIR_LOG}/pb/" ; chmod a+rwx "${DIR_LOG}/pb/"
##########################################################################
DEFAULT_MODE="footer"
@ -53,7 +56,8 @@ EX_UNAVAILABLE=69
EX_TOO_LARGE=552
INSPECT_DIR=/var/spool/filter
DIR_LOG=/var/log/mail
FIC_LOG=${DIR_LOG}/filter.log
FIC_LOG="${DIR_LOG}/filter.log"
TMP_LOG="$(mktemp)"
SENDMAIL="/usr/sbin/sendmail -G -i"
MAILS=/tmp/FILTER
MAX_KEEP_IN_MAIL=5ki
@ -62,73 +66,134 @@ SHRINK_CMD=/home/filter/eMailShrinker
JIRAFEAU_CMD=/home/filter/jirafeauAPI
JIRAFEAU_URL=https://depot.${DOMAINNAME:-"kaz.bzh"}
JIRAFEAU_LOCAL=http://depot
JIRAFEAU_TIME=month
MD5_CMD=/usr/bin/md5sum
DISCLAMER_CMD=altermime
MAX_FINAL_SIZE=2097152 # 2Mi
ARCHIVE_TITLE="archive_content"
ARCHIVE_MIME="text/kaz_email_archive"
FILE_SKIP_DOMAINS="/tmp/docker-mailserver/file_domaines_non_depollues.txt"
#on enlève les commentaires et les lignes vides
SKIP_DOMAINS=`grep -Ev '^#|^[[:space:]]*$' $FILE_SKIP_DOMAINS`
KEEP_FAILED=true
DEBUG=
DEBUG=true
#################### FONCTIONS ############################################
BOLD=''
RED=''
GREEN=''
YELLOW=''
BLUE=''
MAGENTA=''
CYAN=''
NC='' # No Color
BOLD='[1m'
RED='[0;31m'
GREEN='[0;32m'
YELLOW='[0;33m'
BLUE='[0;34m'
MAGENTA='[0;35m'
CYAN='[0;36m'
NC='[0m' # No Color
NL='
'
#--------------------- Fichier de LOG -------------------
LOG_FIC () {
echo "${BLUE}$(date +%d-%m-%Y-%H-%M-%S)${NC} : $*" >> "${FIC_LOG}"
echo "${BLUE}$(date +%d-%m-%Y-%H-%M-%S)${NC} : $*" >> "${TMP_LOG}"
}
quitFilter () {
LOG_FIC "${GREEN}######################################## filter stop${NC}"
cat "${TMP_LOG}" >> "${FIC_LOG}"
rm -f "${TMP_LOG}"
exit $1
}
keepFailed () {
[ -z "${KEEP_FAILED}" ] && return
mkdir -p "${DIR_LOG}/pb/"
cp "$1" "${DIR_LOG}/pb/"
}
########################################
# curl Jirafeau
curlJirafeauUpdate () {
# $1: periode
# $2: jirafeauItemRef
LOG_FIC " - ${CYAN}curl -X POST -d \"u=$1\" -d \"h=$2\" \"${JIRAFEAU_LOCAL}/a.php}\""
curl -X POST -d "u=$1" -d "h=$2" "${JIRAFEAU_LOCAL}/a.php"
}
curlJirafeauSend () {
# $1: periode
# $2: filename
# $3: content-type
# $4: name
# $5: password
type=$3
[ -z "${type}" ] && type="text/plain"
LOG_FIC " - curl -X POST -F \"time=$1\" -F \"key=$5\" -F \"file=@$2;type=${type};filename=\\\"$4\\\"\" \"${JIRAFEAU_LOCAL}/a.php\""
for num in {1..2}; do
OUTPUT=$(curl -X POST -F "time=$1" -F "key=$5" -F "file=@$2;type=${type};filename=\"$4\"" "${JIRAFEAU_LOCAL}/a.php")
read JIR_TOKEN <<< "${OUTPUT}"
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
sleep 30
continue
;;
esac
break
done
echo "${OUTPUT}"
}
# Définir une fonction pour vérifier si le domaine d'un email est dans la liste SKIP_DOMAINS
function check_skip_domains() {
local SKIP_DOMAINS="$1"
local LIST_EMAILS="$2"
for email in $LIST_EMAILS; do
# Extraire le domaine de l'email (partie après le "@")
local domain="${email##*@}"
# Utiliser grep pour vérifier si le domaine est dans la liste des domaines à sauter
if echo "$SKIP_DOMAINS" | grep -q -w "$domain"; then
echo "yes"
return
fi
done
# Si aucun domaine n'a été trouvé, retourner "no"
echo "no"
}
#################### MAIN #################################################
echo "${NL}" >> "${FIC_LOG}"
LOG_FIC "${GREEN}######################################## filter start${NC}"
echo "${NL}${BLUE}$(date +%d-%m-%Y-%H-%M-%S)${NC} : ${GREEN}######################################## filter start (log in ${TMP_LOG})${NC}" >> "${FIC_LOG}"
LOG_FIC "${GREEN}######################################## ${TMP_LOG} ${NC}"
if ! mkdir -p "${MAILS}"; then
LOG_FIC "${RED}Can't mkdir ${MAILS} ${NC}"
quitFilter "${EX_UNAVAILABLE}"
fi
#$@ contient le sender et les destinataires
#format "-f sender -- dest1 [...dest(i)] "
LIST_EMAILS=$(echo $@)
#on nettoie
LIST_EMAILS=$(sed 's/-f//g' <<< $LIST_EMAILS)
LIST_EMAILS=$(sed 's/--//g' <<< $LIST_EMAILS)
MAIL_SOURCE=$(echo $@ | awk 'BEGIN{FS=" "} {print $2}')
DATE_TEMPS=$(date "+%Y-%m-%d-%H:%M:%S")
REP_PIECE_JOINTE="${MAILS}/${DATE_TEMPS}_${MAIL_SOURCE}_$$"
MODE=$(curl "${JIRAFEAU_LOCAL}/a.php?m=${MAIL_SOURCE}" 2>/dev/null )
[[ "${MODE}" =~ ^(footer|attachment|both)$ ]] || MODE="${DEFAULT_MODE}"
[[ "${MODE}" =~ ^(none|footer|attachment|both)$ ]] || MODE="${DEFAULT_MODE}"
TRACK=$(curl "${JIRAFEAU_LOCAL}/a.php?r=${MAIL_SOURCE}" 2>/dev/null )
[[ "${TRACK}" =~ ^(|0|1|false|true|FALSE|TRUE|on|off)$ ]] || TRACK="${DEFAULT_TRACK}"
PERIOD=$(curl "${JIRAFEAU_LOCAL}/a.php?p=${MAIL_SOURCE}" 2>/dev/null )
[[ "${PERIOD}" =~ ^(minute|hour|day|week|month|quarter)$ ]] || PERIOD="${DEFAULT_PERIOD}"
if [ -n "$(echo "${PERIOD}" | grep -e minute -e hour -e day -e week -e month -e quarter 2>/dev/null)" ]; then
JIRAFEAU_TIME="${PERIOD}"
fi
LOG_FIC "\n" \
" MAIL_SOURCE : ${YELLOW}${MAIL_SOURCE}${NC}\n" \
" DATE_TEMPS : ${YELLOW}${DATE_TEMPS=}${NC}\n" \
" TRACK : ${YELLOW}${TRACK}${NC}\n" \
" PERIOD : ${YELLOW}${PERIOD}${NC}\n" \
" JIRAFEAU_TIME: ${YELLOW}${JIRAFEAU_TIME}${NC}"
LOG_FIC "${NL}" \
" MAIL_SOURCE : ${YELLOW}${MAIL_SOURCE}${NC}${NL}" \
" DATE_TEMPS : ${YELLOW}${DATE_TEMPS=}${NC}${NL}" \
" MODE : ${YELLOW}${MODE}${NC}${NL}" \
" TRACK : ${YELLOW}${TRACK}${NC}${NL}" \
" PERIOD : ${YELLOW}${PERIOD}${NC}${NL}"
if ! cd "${INSPECT_DIR}"; then
echo "${INSPECT_DIR} does not exist"
@ -145,41 +210,55 @@ ARCHIVE_CONTENT="${REP_PIECE_JOINTE}/archive-content.txt"
JIRAFEAU_ERROR="${REP_PIECE_JOINTE}/jirafeau-error.txt"
# Clean up when done or when aborting.
[ -z "${DEBUG}" ] && trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
[ -z "${DEBUG}" ] && trap "cd ${INSPECT_DIR}; rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
if ! cat > "in.$$"; then
if ! cat > "${INSPECT_DIR}/in.$$"; then
LOG_FIC "${RED}Cannot save mail to file${NC}"
quitFilter "${EX_TEMPFAIL}"
fi
LOG_FIC "\n" \
dos2unix "${INSPECT_DIR}/in.$$" 2> /dev/null
LOG_FIC "${NL}" \
" size: ${YELLOW}$(wc -c < "${INSPECT_DIR}/in.$$")${NC}"
[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$" "${DIR_LOG}/pb/in.$$.orig")
[ -z "${DEBUG}" ] || (cp "${INSPECT_DIR}/in.$$" "${DIR_LOG}/pb/in.$$.orig")
mkdir -p "${REP_PIECE_JOINTE}/"
>"${OLD_LINKS}"
>"${ARCHIVE_CONTENT}"
if [ "${MODE}" = "none"]; then
LOG_FIC " - ${GREEN}send without change (MODE=none)${NC}"
${SENDMAIL} "$@" < "${INSPECT_DIR}/in.$$"
quitFilter 0
fi
if [ "$(check_skip_domains "$SKIP_DOMAINS" "$LIST_EMAILS")" = "yes" ]; then
LOG_FIC " - ${GREEN}send without change (skip domain)${NC}"
${SENDMAIL} "$@" < "${INSPECT_DIR}/in.$$"
quitFilter 0
fi
# Etape de rafraichissement des anciens fichiers inclus
echo "time: ${DATE_TEMPS}\nid: $(date +%s)" > "${ARCHIVE_CONTENT}"
echo "time: ${DATE_TEMPS}${NL}id: $(date +%s)" > "${ARCHIVE_CONTENT}"
[ -n "${TRACK}" ] && echo "sender: ${MAIL_SOURCE}" >> "${ARCHIVE_CONTENT}"
LOG_FIC "${CYAN}${SHRINK_CMD} -u \"${INSPECT_DIR}/in.$$\" 2>> \"${FIC_LOG}\" > \"${OLD_LINKS}\"${NC}"
"${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" > "${OLD_LINKS}"
LOG_FIC "${CYAN}${SHRINK_CMD} -u \"${INSPECT_DIR}/in.$$\" 2>> \"${TMP_LOG}\" > \"${OLD_LINKS}\"${NC}"
"${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${TMP_LOG}" > "${OLD_LINKS}"
cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
[ -z "${REMOTE_REF}" ] && continue
REMOTE_KEY=$(echo "${REMOTE_LINK}" | grep "k=" | sed 's%.*k=\([^&]*\).*%\1%')
# update periode for download
LOG_FIC " - ${CYAN}\"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -t \"${JIRAFEAU_TIME}\" update \"${REMOTE_REF}\" 2>&1 >> \"${FIC_LOG}\"${NC}"
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -t "${JIRAFEAU_TIME}" update "${REMOTE_REF}" 2>&1 >> "${FIC_LOG}"
curlJirafeauUpdate "${PERIOD}" "${REMOTE_REF}" 2>&1 >> "${TMP_LOG}"
echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "${ARCHIVE_CONTENT}"
done
LOG_FIC " - archive starts with: ${NL}${YELLOW}$(cat ${ARCHIVE_CONTENT})${NC}"
# Etape extraction des pieces jointes
LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_DIR}/in.$$ ${NC}"
"${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" -d "${REP_PIECE_JOINTE}" "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" | {
LOG_FIC "${CYAN}${SHRINK_CMD} -s \"${MAX_KEEP_IN_MAIL}\" -d \"${REP_PIECE_JOINTE}\" \"${INSPECT_DIR}/in.$$\"${NC}"
"${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" -d "${REP_PIECE_JOINTE}" "${INSPECT_DIR}/in.$$" 2>> "${TMP_LOG}" | {
while read ATTACH_TMP_NAME; do
if [ -d "${ATTACH_TMP_NAME}" ]; then
ATTACH_MEDIA="${ATTACH_TMP_NAME}/media"
@ -191,10 +270,9 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
continue
fi
# Etape de televersement des pieces jointes
PASSWORD=$(apg -n 1 -m 12)
PASSWORD=$(apg -n 1 -m 12 -M cln)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
LOG_FIC " - ${CYAN}\"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -t \"${JIRAFEAU_TIME}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ATTACH_CONTENT_TYPE}\" -n \"${ATTACH_NAME}\" send \"${ATTACH_MEDIA}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\"${NC}"
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -t "${JIRAFEAU_TIME}" -s "${MAX_UPLOAD_SIZE}" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}"
curlJirafeauSend "${PERIOD}" "${ATTACH_MEDIA}" "${ATTACH_CONTENT_TYPE}" "${ATTACH_NAME}" "${PASSWORD}" 2>> "${TMP_LOG}" > "${ONE_LINK}"
cat "${ONE_LINK}" | {
read JIR_TOKEN
read JIR_CODE
@ -207,8 +285,8 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
echo "UPLOAD_FAIL" >> "${JIRAFEAU_ERROR}"
;;
* )
LOG_FIC " - change by link ${YELLOW}${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}${NC}"
echo "url: ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
LOG_FIC " - change by link ${YELLOW}${JIRAFEAU_URL}/f.php?d=0&h=${JIR_TOKEN}&k=${PASSWORD_MD5}${NC}"
echo "url: ${JIRAFEAU_URL}/f.php?d=0&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "${ARCHIVE_CONTENT}"
;;
esac
@ -217,11 +295,11 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
# Création de l'archive
NB_ATTACH=$(grep -e "^old: " -e "^new: " "${ARCHIVE_CONTENT}" | wc -l)
if [ \( -n "${TRACK}" -a "${NB_ATTACH}" -gt 0 \) -o "${NB_ATTACH}" -gt 1 ]; then
PASSWORD=$(apg -n 1 -m 12)
PASSWORD=$(apg -n 1 -m 12 -M cln)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
LOG_FIC " - ${MAGENTA}upload archive${NC}"
LOG_FIC " - ${CYAN}\"${JIRAFEAU_CMD}\" -f \"${JIRAFEAU_LOCAL}\" -t \"${JIRAFEAU_TIME}\" -s \"${MAX_UPLOAD_SIZE}\" -c \"${ARCHIVE_MIME}\" -n \"${ARCHIVE_TITLE}\" send \"${ARCHIVE_CONTENT}\" \"${PASSWORD}\" 2>> \"${FIC_LOG}\" > \"${ONE_LINK}\"${NC}"
"${JIRAFEAU_CMD}" -f "${JIRAFEAU_LOCAL}" -t "${JIRAFEAU_TIME}" -s "${MAX_UPLOAD_SIZE}" -c "${ARCHIVE_MIME}" -n "${ARCHIVE_TITLE}" send "${ARCHIVE_CONTENT}" "${PASSWORD}" 2>> "${FIC_LOG}" > "${ONE_LINK}"
curlJirafeauSend "${PERIOD}" "${ARCHIVE_CONTENT}" "${ARCHIVE_MIME}" "${ARCHIVE_TITLE}" "${PASSWORD}" 2>> "${TMP_LOG}" > "${ONE_LINK}"
fi
LOG_FIC " - final archive content: ${NL}${YELLOW}$(cat ${ARCHIVE_CONTENT})${NC}"
if [ "${NB_ATTACH}" -gt 1 ]; then
@ -247,10 +325,10 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
echo "arch: none"
fi
# Etape de substitution
LOG_FIC "${CYAN}${SHRINK_CMD} -m \"${MODE}\" -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"${NC}"
} | "${SHRINK_CMD}" -m "${MODE}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
LOG_FIC "${CYAN}${SHRINK_CMD} -m \"${MODE}\" -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${TMP_LOG}\"${NC}"
} | "${SHRINK_CMD}" -m "${MODE}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${TMP_LOG}"
[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$.altered" "${DIR_LOG}/pb/in.$$.altered")
[ -z "${DEBUG}" ] || (cp "${INSPECT_DIR}/in.$$.altered" "${DIR_LOG}/pb/in.$$.altered")
if [ -s "${JIRAFEAU_ERROR}" ]; then
LOG_FIC " - ${RED}upload fail${NC}"
@ -261,7 +339,7 @@ fi
if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 3 ]; then
# verification de taille finale
actualSize=$(wc -c < "${INSPECT_DIR}/in.$$.altered")
if [ ${actualSize} -ge $MAX_FINAL_SIZE ]; then
if [ "${actualSize}" -ge "${MAX_FINAL_SIZE}" ]; then
LOG_FIC " - ${RED}too big even after diet ${INSPECT_DIR}/in.$$.altered (${actualSize})${NC}"
keepFailed "${INSPECT_DIR}/in.$$"
quitFilter "${EX_TOO_LARGE}"
@ -271,15 +349,16 @@ if [ "$(wc -l < "${ARCHIVE_CONTENT}")" -ge 3 ]; then
else
# verification de taille finale
actualSize=$(wc -c < "${INSPECT_DIR}/in.$$")
if [ ${actualSize} -ge $MAX_FINAL_SIZE ]; then
if [ "${actualSize}" -ge "${MAX_FINAL_SIZE}" ]; then
LOG_FIC " - ${RED}too big without diet ${INSPECT_DIR}/in.$$ (${actualSize}) ${NC}"
keepFailed "${INSPECT_DIR}/in.$$"
quitFilter "${EX_TOO_LARGE}"
fi
LOG_FIC " - ${GREEN}send without attach file${NC}"
${SENDMAIL} "$@" < "in.$$"
${SENDMAIL} "$@" < "${INSPECT_DIR}/in.$$"
fi
quitFilter 0
##########################################################################

166
src/bash/filterTest.sh Normal file → Executable file
View File

@ -1,8 +1,41 @@
#!/bin/bash
##########################################################################
# Copyright KAZ 2021 #
# #
# contact (at) kaz.bzh #
# #
# This software is a filter to shrink email by attachment extraction. #
# #
# This software is governed by the CeCILL-B license under French law and #
# abiding by the rules of distribution of free software. You can use, #
# modify and/or redistribute the software under the terms of the #
# CeCILL-B license as circulated by CEA, CNRS and INRIA at the following #
# URL "http://www.cecill.info". #
# #
# As a counterpart to the access to the source code and rights to copy, #
# modify and redistribute granted by the license, users are provided #
# only with a limited warranty and the software's author, the holder of #
# the economic rights, and the successive licensors have only limited #
# liability. #
# #
# In this respect, the user's attention is drawn to the risks associated #
# with loading, using, modifying and/or developing or reproducing the #
# software by the user in light of its specific status of free software, #
# that may mean that it is complicated to manipulate, and that also #
# therefore means that it is reserved for developers and experienced #
# professionals having in-depth computer knowledge. Users are therefore #
# encouraged to load and test the software's suitability as regards #
# their requirements in conditions enabling the security of their #
# systems and/or data to be ensured and, more generally, to use and #
# operate it in the same conditions as regards security. #
# #
# The fact that you are presently reading this means that you have had #
# knowledge of the CeCILL-B license and that you accept its terms. #
##########################################################################
PRG=$(basename $0)
ATTACH_MODE="-m BOTH"
ATTACH_MODE="FOOTER"
BOLD=''
RED=''
@ -16,55 +49,110 @@ NL='
'
TTY=$(tty)
########################################
LOG () {
echo "$1" >> "${TTY}"
}
usage () {
echo "Usage: ${PRG} mbox"
echo "Usage: ${PRG} [-h|-v|-g] [-m {NONE|FOOTER|ATTACHMENT|BOTH}] mbox"
exit 1
}
while : ; do
case "$1" in
-h*) usage;;
-v*) "${eMailShrinker}" -v; exit;;
-g) DEBUG="-g"; shift;;
-m) shift; ATTACH_MODE="$1"; shift;;
*) break;;
esac
done
case "${ATTACH_MODE}" in
""|NONE|FOOTER|ATTACHMENT|BOTH);;
*) usage;;
esac
[ -z "${ATTACH_MODE}" ] || ATTACH_MODE="-m ${ATTACH_MODE}"
[ "$#" -eq 1 ] || usage
mbox=$(realpath "$1")
mbox=$(realpath $1)
########################################
# recherche des binaires
cd $(dirname $0)
eMailShrinker="$(realpath "./eMailShrinker")"
[ -x "${eMailShrinker}" ] || eMailShrinker="$(realpath "../../build/out/eMailShrinker")"
[ -x "${eMailShrinker}" ] || ( echo "${RED}eMailShrinker not found${NC}" ; exit)
########################################
dos2unix "${mbox}"
DOMAINNAME="$(cat domainname)"
JIRAFEAU_URL="https://depot.${DOMAINNAME}"
JIRAFEAU_LOCAL="${JIRAFEAU_URL}"
cd ../..
TMP_DIR="$(mktemp)"
mkdir -p tmp
rm -fr tmp/PJ-name.txt tmp/PJ-Keys.txt tmp/PJ tmp/archive-content.txt tmp/url-to-refresh.txt tmp/new-mbox
echo -e "time: $(date "+%Y-%m-%d-%H:%M:%S")\nid: $(date +%s)" > "tmp/archive-content.txt"
########################################
# curl Jirafeau
curlJirafeauUpdate () {
# $1: periode
# $2: jirafeauItemRef
curl -X POST -d "u=$1" -d "h=$2" "${JIRAFEAU_LOCAL}/a.php"
#"${jirafeauAPI}" -f "${JIRAFEAU_LOCAL}" -t "$1" update "$2"
}
curlJirafeauSend () {
# $1: periode
# $2: filename
# $3: content-type
# $4: name
# $5: password
type=$3
[ -z "${type}" ] && type="text/plain"
LOG "curl -X POST -F \"time=$1\" -F \"key=$5\" -F \"file=@$2;type=${type};filename=\\\"$4\\\"\" \"${JIRAFEAU_LOCAL}/a.php\""
curl -X POST -F "time=$1" -F "key=$5" -F "file=@$2;type=${type};filename=\"$4\"" "${JIRAFEAU_LOCAL}/a.php" || exit 1
#"${jirafeauAPI}" -f "${JIRAFEAU_LOCAL}" -t "$1" -s "1Gi" -c "${type}" -n "$4" send "$2" "$5"
}
########################################
# nettoyage
rm -f "${TMP_DIR}" ; mkdir -p "${TMP_DIR}"
rm -fr "${TMP_DIR}/PJ-name.txt" "${TMP_DIR}/PJ-Keys.txt" "${TMP_DIR}/PJ" "${TMP_DIR}/archive-content.txt" "${TMP_DIR}/url-to-refresh.txt" "${TMP_DIR}/new-mbox"
echo -e "time: $(date "+%Y-%m-%d-%H:%M:%S")\nid: $(date +%s)" > "${TMP_DIR}/archive-content.txt"
########################################
# affichage de la structure de départ
LOG " - ${BLUE}mbox: ${mbox}${NC}"
build/out/eMailShrinker -l "${mbox}"
"${eMailShrinker}" -l "${mbox}"
LOG
build/out/eMailShrinker -u "${mbox}" > "tmp/url-to-refresh.txt" 2>> "${TTY}"
cat "tmp/url-to-refresh.txt" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
########################################
# recherche des prolongations des délais de grace
"${eMailShrinker}" ${DEBUG} -u "${mbox}" > "${TMP_DIR}/url-to-refresh.txt" 2>> "${TTY}"
cat "${TMP_DIR}/url-to-refresh.txt" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
[ -z "${REMOTE_REF}" ] && continue
LOG " - ${BLUE}update ${REMOTE_REF}${NC}"
build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" update "${REMOTE_REF}" 2>> "${TTY}"
curlJirafeauUpdate "month" "${REMOTE_REF}" 2>> "${TTY}"
LOG
echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "tmp/archive-content.txt"
echo "old: ${REMOTE_REF} ${REMOTE_KEY}" >> "${TMP_DIR}/archive-content.txt"
done
build/out/eMailShrinker -s "5ki" -d tmp/PJ "${mbox}" > "tmp/PJ-name.txt"
########################################
# extraction des pièces jointes
"${eMailShrinker}" ${DEBUG} -s "5ki" -d "${TMP_DIR}/PJ" "${mbox}" > "${TMP_DIR}/PJ-name.txt" || exit 1
LOG " - ${BLUE}PJ-name: ${NC}"
cat tmp/PJ-name.txt
cat "${TMP_DIR}/PJ-name.txt"
LOG
cat "tmp/PJ-name.txt" | {
########################################
# dépot des extractions dans jirafeau et récupération des codes
cat "${TMP_DIR}/PJ-name.txt" | {
while read ATTACH_TMP_NAME; do
LOG " - ${BLUE}find ATTACH_TMP_NAME: (${ATTACH_TMP_NAME}) ${NC}"
if [ -d "${ATTACH_TMP_NAME}" ]; then
@ -77,39 +165,41 @@ cat "tmp/PJ-name.txt" | {
continue
fi
LOG " - ${BLUE}find ${ATTACH_NAME} / (${ATTACH_CONTENT_TYPE}) / ${ATTACH_MEDIA} ${NC}"
PASSWORD=$(apg -n 1 -m 12)
PASSWORD=$(apg -n 1 -m 12 -M cln)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" -s "1Gi" -c "${ATTACH_CONTENT_TYPE}" -n "${ATTACH_NAME}" send "${ATTACH_MEDIA}" "${PASSWORD}" 2>> "${TTY}" > "tmp/one.txt"
curlJirafeauSend "month" "${ATTACH_MEDIA}" "${ATTACH_CONTENT_TYPE}" "${ATTACH_NAME}" "${PASSWORD}" 2>> "${TTY}" > "${TMP_DIR}/one.txt"
cat "tmp/one.txt" | {
cat "${TMP_DIR}/one.txt" | {
read JIR_TOKEN
read JIR_CODE
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
LOG " - ${RED}can't upload ${ATTACH_MEDIA} <${JIR_TOKEN}> <${JIR_CODE}>${NC}"
cat "tmp/one.txt" >> "${TTY}"
cat "${TMP_DIR}/one.txt" >> "${TTY}"
echo "url:"
exit 1
;;
* )
LOG " - ${GREEN} upload ${ATTACH_MEDIA}${NC}"
echo "url: ${JIRAFEAU_URL}/f.php?d=1&h=${JIR_TOKEN}&k=${PASSWORD_MD5}"
echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "tmp/archive-content.txt"
echo "new: ${JIR_TOKEN} ${PASSWORD_MD5}" >> "${TMP_DIR}/archive-content.txt"
;;
esac
}
done
NB_ATTACH=$(grep -e "^old: " -e "^new: " "tmp/archive-content.txt" | wc -l)
NB_ATTACH=$(grep -e "^old: " -e "^new: " "${TMP_DIR}/archive-content.txt" | wc -l)
if [ "${NB_ATTACH}" -gt 1 ]; then
PASSWORD=$(apg -n 1 -m 12)
PASSWORD=$(apg -n 1 -m 12 -M cln)
PASSWORD_MD5=$(echo -n ${PASSWORD} | ${MD5_CMD} | cut -d \ -f 1)
build/out/jirafeauAPI -f "${JIRAFEAU_LOCAL}" -t "month" -s "1Gi" -c "text/kaz_email_archive" -n "archive_content" send "tmp/archive-content.txt" "${PASSWORD}" > "tmp/one.txt" 2>> "${TTY}"
cat "tmp/one.txt" | {
curlJirafeauSend "month" "${TMP_DIR}/archive-content.txt" "text/kaz_email_archive" "archive_content" "${PASSWORD}" 2>> "${TTY}" > "${TMP_DIR}/one.txt" || exit 1
cat "${TMP_DIR}/one.txt" | {
read JIR_TOKEN
read JIR_CODE
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
LOG " - ${RED}can't upload tmp/archive-content.txt${NC}"
LOG " - ${RED}can't upload ${TMP_DIR}/archive-content.txt${NC}"
echo "arch: bad"
exit 1
;;
* )
LOG " - ${GREEN} upload archive-content.txt${NC}"
@ -121,16 +211,22 @@ cat "tmp/PJ-name.txt" | {
LOG " - ${GREEN}no archive${NC}"
echo "arch: none"
fi
} > "tmp/PJ-Keys.txt"
} > "${TMP_DIR}/PJ-Keys.txt"
LOG " - ${BLUE}PJ-Keys: ${NC}"
cat tmp/PJ-Keys.txt
cat "${TMP_DIR}/PJ-Keys.txt"
LOG
LOG " - ${GREEN}ATTACH_MODE: ${ATTACH_MODE}${NC}"
cat "tmp/PJ-Keys.txt" | build/out/eMailShrinker ${ATTACH_MODE} -s "5ki" "${mbox}" "tmp/new-mbox" 2>> "${TTY}"
########################################
# substitution des pièces jointes par les codes fournis par jirafeau
cat "${TMP_DIR}/PJ-Keys.txt" | "${eMailShrinker}" ${DEBUG} ${ATTACH_MODE} -s "5ki" "${mbox}" "${TMP_DIR}/new-mbox" 2>> "${TTY}" || exit 1
########################################
# affichage de la structure à la fin
LOG " - ${BLUE}new-mbox:${NC}"
build/out/eMailShrinker -l "tmp/new-mbox" 2>> "${TTY}"
"${eMailShrinker}" -l "${TMP_DIR}/new-mbox" 2>> "${TTY}" || exit 1
echo -e "\nresul in ${TMP_DIR}/new-mbox"
exit 0

View File

@ -60,23 +60,17 @@ const string Attachment::contentIDToken ("content-id");
const string Attachment::PLAIN ("plain");
const string Attachment::HTML ("html");
const string Attachment::RELATED ("related");
const string Attachment::SIGNED ("signed");
const string Attachment::ALTERNATIVE ("alternative");
const string Attachment::KAZ_ATTACH_NAME (".---KazAttachment---.html");
const string Attachment::KAZ_ATTACH_NAME ("vos-pieces-jointes-kaz-ici.htm");
const string Attachment::MULTIPART ("multipart/");
const regex Attachment::nameCharsetRegEx (".*name\\*=(.*)");
const regex Attachment::nameRegEx (".*name=\\s*\"?(.*)\"?\\s*;?\\s*");
// boundary="----=_Part_796779_1154936629.1668080348646"
// boundary="------------040709000505010508040808"
// boundary="----------=_1668606031-941125-91"
// boundary="_004_PAVPR10MB6792713B313048E3A259B215B2079PAVPR10MB6792EURP_";
// boundary="_000_PAVPR10MB6792713B313048E3A259B215B2079PAVPR10MB6792EURP_"
// boundary=--boundary_1351_64006126-2b0e-4a3b-98ac-4797d1634188
// boundary=--boundary_1352_7e294c9a-cfab-44a0-bfb3-7310380ac7cb;
const regex Attachment::boundaryRegEx (".*boundary=\"?([^\"; ]*)\"?;?.*");
const regex Attachment::nameCharsetRegEx ( ".*name\\*=\\s*([; \t]*)");
const regex Attachment::nameRegEx ( ".*name=\\s*((\"(\\\\.|[^\\\\])*\")|[^; \t]*).*");
const regex Attachment::boundaryRegEx (".*boundary=\\s*((\"(\\\\.|[^\\\\])*\")|[^; \t]*).*");
const regex Attachment::cidDefRegEx (".*<([^>]*)>.*");
const regex Attachment::textRegEx (".*text/("+PLAIN+"|"+HTML+").*");
const regex Attachment::multiRegEx ("\\s*multipart/(mixed|"+RELATED+"|"+ALTERNATIVE+").*");
const regex Attachment::multiRegEx ("\\s*"+MULTIPART+"(mixed|"+RELATED+"|"+ALTERNATIVE+"|"+SIGNED+").*");
const string Attachment::IMG_BEGIN ("<IMG");
const string Attachment::IMG_END (">");
@ -156,6 +150,7 @@ Attachment::getSection (const string &content, const string &beginTag, const str
string::size_type stopPos = caseInsensitiveFind (content, endTag, startPos);
LOG_BUG (stopPos == string::npos, break, "eMailShrinker: bug A3: " << endTag << " not found! at: " << startPos);
// LOG_BUG (stopPos == string::npos, break, "eMailShrinker: bug A3: " << endTag << " not found! at: " << startPos << endl << content);
LOG ("start: " << startPos << " stop: " << stopPos);
LOG_BUG (startPos == stopPos, /**/, "eMailShrinker: bug A4: " << endTag << " without " << beginTag << " at: " << startPos);
@ -182,32 +177,51 @@ Attachment::getContentType () const {
const string
Attachment::getAttachName () const {
static string tokens [] = {contentTypeToken, contentDispositionToken};
DEF_LOG ("Attachment::getAttachName", "");
string result = getProp (contentTypeToken, nameRegEx);
for (string token : tokens) {
// name=
string result = getProp (token, nameRegEx);
removeQuote (result);
if (result.length ()) {
LOG ("name=: " << result);
encodedWord (result);
encodedWordDecode (result);
return result;
}
result = getProp (contentTypeToken, nameCharsetRegEx);
// name*x=
for (int id = 0; ; ++id) {
string item = getProp (token, regex (".*name\\*"+to_string (id)+"=\\s*((\"(\\\\.|[^\\\\])*\")|[; \t]*).*"));
if (item.empty ())
break;
result += item;
}
removeQuote (result);
if (result.length ()) {
LOG ("name*x=: " << result);
encodedWordDecode (result);
return result;
}
// name*=
result = getProp (token, nameCharsetRegEx);
removeQuote (result);
if (result.length ()) {
LOG ("name*=: " << result);
charsetValue (result);
charsetValueDecode (result);
return result;
}
// XXX il faut composer s'il y a plusieurs ligne filename*x=
result = getProp (contentDispositionToken, nameRegEx);
// name*x*=
for (int id = 0; ; ++id) {
string item = getProp (token, regex (".*name\\*"+to_string (id)+"\\*=\\s*([^; ]*)"));
if (item.empty ())
break;
result += item;
}
removeQuote (result);
if (result.length ()) {
LOG ("filename=: " << result);
encodedWord (result);
LOG ("name*x*=: " << result);
encodedWordDecode (result);
return result;
}
// XXX il faut composer s'il y a plusieurs ligne filename*x*=
result = getProp (contentDispositionToken, nameRegEx);
if (result.length ()) {
LOG ("filename*=: " << result);
charsetValue (result);
return result;
}
return getUnknown (getContentType ());
}
@ -261,7 +275,7 @@ Attachment::isDefProp (const string &token, const string &val) const {
if (it == env.end ())
return false;
// XXX case insensitive ??
return it->second.find (val) != string::npos;
return caseInsensitiveFind (it->second, val) != string::npos;
}
// ================================================================================
@ -275,6 +289,7 @@ Attachment::Attachment (ifstream &mbox, const int &level, const streamoff beginI
toUpdate (false),
toDisclaim (false),
isKazAttachment (false),
isSigned (false),
boundaryMiddleSize (0) {
DEF_LOG ("Attachment::Attachment", "curPos: " << curPos << " level: " << level);
readMime (mbox, curPos);
@ -282,6 +297,13 @@ Attachment::Attachment (ifstream &mbox, const int &level, const streamoff beginI
}
// ================================================================================
inline string
cleanString (const string &line) {
if (!line.empty () && line[line.size() - 1] == '\r')
return line.substr (0, line.size () - 1);
return line;
}
void
Attachment::readMime (ifstream &mbox, streamoff &curPos) {
DEF_LOG ("Attachment::readMime", "curPos: " << curPos);
@ -290,7 +312,7 @@ Attachment::readMime (ifstream &mbox, streamoff &curPos) {
for (; getline (mbox, line); ) {
LOG ("pos: " << curPos << " line: " << line);
curPos += line.length () + 1;
if (line.empty ())
if (line.empty () || "\r" == line)
break;
if (line[0] == ' ' || line[0] == '\t') {
if (lastVar.empty ()) {
@ -298,8 +320,8 @@ Attachment::readMime (ifstream &mbox, streamoff &curPos) {
LOG_BUG (true, /**/, "eMailShrinker: bug A5: not compliant MIME. pos: " << (curPos - (line.length () + 1)) << " line: " << line);
} else {
LOG ("add line to var: " << line);
env.find (lastVar)->second += line;
LOG ("new val: " << env.find (lastVar)->second);
env.find (lastVar)->second += cleanString (line);
LOG ("new val: <" << lastVar << " <=> " << env.find (lastVar)->second << ">");
}
continue;
}
@ -308,8 +330,11 @@ Attachment::readMime (ifstream &mbox, streamoff &curPos) {
lastVar = line.substr (0, colonPos);
toLower (lastVar);
LOG ("find var: " << lastVar);
string val (line.length () >= colonPos+2 ? line.substr (colonPos+2) : ""); // XXX check RFC " " after ": "
LOG ("new var: " << lastVar << " <=> " << val);
++colonPos;
if (line.length () >= colonPos && line [colonPos] == ' ')
++colonPos;
string val (cleanString (line.length () >= colonPos ? line.substr (colonPos) : ""));
LOG ("new var: <" << lastVar << " <=> " << val << ">");
env [lastVar] = val;
}
}
@ -317,12 +342,19 @@ Attachment::readMime (ifstream &mbox, streamoff &curPos) {
contentPos = curPos;
cid = getProp (contentIDToken, cidDefRegEx);
if (caseInsensitiveFind (getContentType (), MULTIPART) != string::npos) {
string multiProp = getProp (contentTypeToken, multiRegEx);
if (SIGNED == multiProp)
isSigned = true;
boundary = getProp (contentTypeToken, boundaryRegEx);
removeQuote (boundary);
LOG ("boundary: " << boundary);
if (boundary.length ()) {
boundary = "--"+boundary+"--";
boundaryMiddleSize = boundary.length () - 2;
}
}
LOG ("readMime contentPos: " << contentPos << " cid: " << cid << " boundary: " << boundary);
}
@ -410,18 +442,19 @@ Attachment::markDisclaim (bool &plainMarked, bool &htmlMarked) {
// ================================================================================
bool
Attachment::markSignificant (const string &parentMultiProp, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs) {
Attachment::markSignificant (const string &parentMultiProp, const bool &parentSigned, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs) {
DEF_LOG ("Attachment::markSignificant", "parentMultiProp: " << parentMultiProp << " minAttachSize: " << minAttachSize);
isSigned |= parentSigned;
string textProp = getProp (contentTypeToken, textRegEx);
bool cantBeExtract ((parentMultiProp == ALTERNATIVE && (textProp == PLAIN || textProp == HTML)) ||
(parentMultiProp == RELATED && textProp == HTML));
string multiProp = getProp (contentTypeToken, multiRegEx);
for (Attachment &sub : subAttachements)
cantBeExtract |= sub.markSignificant (multiProp, minAttachSize, mbox, allMarkedPtrs);
cantBeExtract |= sub.markSignificant (multiProp, parentSigned || isSigned, minAttachSize, mbox, allMarkedPtrs);
if (getProp (contentTypeToken, textRegEx) == HTML) {
if (KAZ_ATTACH_NAME == getAttachName ()) {
if (KAZ_ATTACH_NAME == getAttachName ())
isKazAttachment = true;
} else {
else {
string content = getContent (mbox);
vector<string> imgs;
getSection (content, IMG_BEGIN, IMG_END, imgs);
@ -432,16 +465,27 @@ Attachment::markSignificant (const string &parentMultiProp, const streamoff &min
}
cantBeExtract |= toUpdate;
if (boundary.empty () && getSize () >= minAttachSize && !cantBeExtract)
cantBeExtract = toExtract = true; // XXX cantBeExtract ?
cantBeExtract = toExtract = true;
if (toExtract || toUpdate || toDisclaim || isKazAttachment)
allMarkedPtrs.push_back (this);
return cantBeExtract;
}
// ================================================================================
string
Attachment::getMime (ifstream &mbox) const {
DEF_LOG ("Attachment::getMime", "beginPos: " << beginPos << " contentPos: " << contentPos);
string mime;
mime.resize (contentPos-beginPos);
mbox.seekg (beginPos, ios::beg);
mbox.read (&mime[0], contentPos-beginPos);
return mime;
}
// ================================================================================
string
Attachment::getContent (ifstream &mbox) const {
DEF_LOG ("Attachment::getContent", "contentPos: " << contentPos);
DEF_LOG ("Attachment::getContent", "contentPos: " << contentPos << " endPos: " << endPos);
string content;
content.resize (endPos-contentPos);
mbox.seekg (contentPos, ios::beg);
@ -499,7 +543,9 @@ Attachment::replaceEmbedded (string &content) const {
ostream&
kaz::operator << (ostream& os, const Attachment& attachment) {
string prop, sep;
if (attachment.toExtract) { prop = "to extract"; sep = ", "; }
if (attachment.isSigned) { prop += sep+"signed"; sep = ", "; }
if (attachment.isKazAttachment) { prop += sep+"kazDisclaim"; sep = ", "; }
if (attachment.toExtract) { prop += sep+"to extract"; sep = ", "; }
if (attachment.toUpdate) { prop += sep+"need update"; sep = ", "; }
if (attachment.toDisclaim) { prop += sep+"need diclaim"; sep = ", "; }
if (attachment.embeddedData.size ()) { prop += sep+"embeddedData"; }

View File

@ -45,6 +45,7 @@
#include <boost/assign.hpp>
#include <boost/algorithm/string.hpp>
#include "version.hpp"
#include "kazDebug.hpp"
#include "kazMisc.hpp"
#include "SizeArg.hpp"
@ -60,13 +61,12 @@ static const string TMPL_FILENAME ("{{FILENAME}}");
static const string CID ("cid:");
// "l=/" => v1 compatibility
static const regex archiveURLSignature (".*(([&?]g=)|([&?]l=/)).*");
static const regex archiveURLRegex (".*(([&?]g=)|([&?]l=/)).*");
static const string KAZ_PLAIN_HR ("______________________________________________________________________________");
static const string KAZ_PLAIN_START ("~~ PJ-KAZ !"); // don't end whith space
static const string KAZ_PLAIN_STOP (KAZ_PLAIN_START+" ~~");
static const string KAZ_PLAIN_DONT_TOUCH ("(conservez cette partie intacte dans votre réponse si vous voulez transmettre les documents précédents)");
static const string KAZ_PLAIN_DONT_TOUCH ("(conservez cette partie intacte dans votre réponse si vous voulez transmettre les documents précédents (version "+LAST_VERSION_NUM+"))");
static const string KAZ_PLAIN_WARNING ("Attention : Kaz a dépollué ce message. Les pièces jointes ont été retirées et placées dans un dépôt provisoire. "
"Elles seront automatiquement supprimées dans 1 mois. "
"Si elles sont importantes et que vous souhaitez les conserver, vous devez utiliser les liens ci-dessous. "
@ -94,7 +94,7 @@ static const string KAZ_HTML_TAG ("<!--KAZ"); // don't end whith space
static const string KAZ_HTML_START (KAZ_HTML_TAG+" START-->");
static const string KAZ_HTML_STOP (KAZ_HTML_TAG+" STOP-->");
// Textes précédents encodés en SGML
static const string KAZ_HTML_DONT_TOUCH ("(conservez cette partie intacte dans votre r&eacute;ponse si vous voulez transmettre les documents pr&eacute;c&eacute;dents)");
static const string KAZ_HTML_DONT_TOUCH ("(conservez cette partie intacte dans votre r&eacute;ponse si vous voulez transmettre les documents pr&eacute;c&eacute;dents (version "+LAST_VERSION_NUM+"))");
static const string KAZ_HTML_DOWLOAD_ONE ("Vos pi&egrave;ces jointes sont &agrave; t&eacute;l&eacute;charger individuellement ici :");
static const string KAZ_HTML_DOWLOAD_OTHER ("(Contenu dans des messages pr&eacute;c&eacute;dents)");
static const string KAZ_HTML_DOWLOAD_ALL ("Vous pouvez t&eacute;l&eacute;charger l'ensemble dans une archive l&agrave; :");
@ -112,8 +112,8 @@ vector <string>
Attachment::stringsToUpdate ({KAZ_PLAIN_START, "\""+CID});
// ================================================================================
const string MainAttachment::templatePlainAddLink (" * "+TMPL_FILENAME+" <"+TMPL_DOWNLOAD+">\r\n");
const string MainAttachment::templatePlainAllLink ("\r\n * "+KAZ_PLAIN_DOWLOAD_ALL+" <"+TMPL_DOWNLOAD+">\r\n");
const string MainAttachment::templatePlainAddLink (" * "+TMPL_FILENAME+" < "+TMPL_DOWNLOAD+" >\n");
const string MainAttachment::templatePlainAllLink ("\n * "+KAZ_PLAIN_DOWLOAD_ALL+" < "+TMPL_DOWNLOAD+" >\n");
const string MainAttachment::templateHtmlHeader (KAZ_HTML_START+"<p style=\"clear: left; padding: 1pc 0 0 0; font-size:10px; color:#969696;\">"+KAZ_PLAIN_START+"</p><hr>\n"
"<div class=\"kaz\">"
@ -124,7 +124,7 @@ const string MainAttachment::templateHtmlOtherLink ("</ul>"+KAZ_HTML_DOWLOAD_OTH
const string MainAttachment::templateHtmlAllLink ("</ul><ul>"+LI_ALL+KAZ_HTML_DOWLOAD_ALL+" <a href=\""+TMPL_DOWNLOAD+"\">"+KAZ_HTML_ARCHIVE+"</a>"+LI_END+"\n");
const string MainAttachment::templateHtmlFooter ("</ul></p>\n"
"<p class=\"msg\"><a class=\"kaz\" href=\""+KAZ_WEB_SITE+"\"> "+KAZ_WEB_SITE+" </a></p></div>\n"
"<hr><p style=\"font-size:10px; color:#969696;\">"+KAZ_PLAIN_STOP+"</p>"+KAZ_HTML_STOP+"\n");
"<hr><p style=\"font-size:10px; color:#969696;\">"+KAZ_PLAIN_STOP+"</p>"+KAZ_HTML_STOP+"\n\n");
const regex MainAttachment::whiteSpaceRegEx ("\\s+");
@ -157,6 +157,35 @@ kaz::operator >> (istream &in, AttachMode &attachMode) {
return in;
}
// ================================================================================
const string
kaz::headerTypeLabels[] = {
"Same", "Multi", "MainPlain", "AttachHtml"
};
const map<string, HeaderType>
kaz::headerTypeMap = boost::assign::map_list_of
("same", SAME)
("multi", MULTI)
("mainplain", MAIN_PLAIN)
("attachhtml", ATTACH_HTML)
;
ostream &
kaz::operator << (ostream &out, const HeaderType &headerType) {
//BOOST_ASSERT (treeType >= MIN && treeType <= ALPHA);
return out << headerTypeLabels [headerType];
}
istream &
kaz::operator >> (istream &in, HeaderType &headerType) {
string token;
in >> token;
auto pos = headerTypeMap.find (boost::algorithm::to_lower_copy (token));
if (pos == headerTypeMap.end ())
in.setstate (ios_base::failbit);
else
headerType = pos->second;
return in;
}
// ================================================================================
void
MainAttachment::copy (ifstream &mbox, ofstream &outbox, const streamoff &begin, const streamoff &end) {
@ -177,8 +206,8 @@ MainAttachment::readArchiveUrl () {
archiveDownloadURL.clear ();
string line;
getline (cin, line);
LOG_BUG (line.rfind ("arch: ", 0) != 0, return, "eMailShrinker: bug ZZ: no archive link. (line: " << line << ")");
LOG_BUG (line.rfind ("arch: bad", 0) == 0, return, "eMailShrinker: bug ZZ: bad archive link. (line: " << line << ")");
LOG_BUG (line.rfind ("arch: ", 0) != 0, return, "eMailShrinker: bug M9: no archive link. (line: " << line << ")");
LOG_BUG (line.rfind ("arch: bad", 0) == 0, return, "eMailShrinker: bug M10: bad archive link. (line: " << line << ")");
if (line.rfind ("arch: none", 0) == 0)
return;
archiveDownloadURL = line.substr (6);
@ -192,7 +221,7 @@ MainAttachment::readDownloadUrl (string &url) {
string line;
getline (cin, line);
LOG ("get URL: " << line);
LOG_BUG (line.rfind ("url: ", 0) != 0, return, "eMailShrinker: bug ZZ: no download link. (line: " << line << ")");
LOG_BUG (line.rfind ("url: ", 0) != 0, return, "eMailShrinker: bug M11: no download link. (line: " << line << ")");
url = line.substr (5);
}
@ -231,7 +260,7 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
int linkCount (0);
string plainNewLinks, htmlNewLinks;
for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->toExtract)
if (attachP->isSigned || !attachP->toExtract)
continue;
addLink (plainNewLinks, htmlNewLinks, attachP->downloadUrl, attachP->getAttachName ());
++linkCount;
@ -240,7 +269,7 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
// previousLinks.erase (attachP->downloadUrl);
}
for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ())
if (attachP->isSigned || !attachP->embeddedData.size ())
continue;
for (EmbeddedData &embedded : attachP->embeddedData) {
addLink (plainNewLinks, htmlNewLinks, embedded.downloadUrl, embedded.name);
@ -257,10 +286,10 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
return;
}
plain = "\r\n"+KAZ_PLAIN_START+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_DONT_TOUCH+"\r\n\r\n"+KAZ_PLAIN_DOWLOAD_ONE+"\r\n"+plainNewLinks;
plain = "\n"+KAZ_PLAIN_START+"\n\n"+KAZ_PLAIN_HR+"\n"+KAZ_PLAIN_DONT_TOUCH+"\n\n"+KAZ_PLAIN_DOWLOAD_ONE+"\n"+plainNewLinks;
html = templateHtmlHeader+htmlNewLinks;
if (previousLinks.size ()) {
plain += "\r\n"+KAZ_PLAIN_DOWLOAD_OTHER+"\r\n"+plainOldLinks;
plain += "\n"+KAZ_PLAIN_DOWLOAD_OTHER+"\n"+plainOldLinks;
html += templateHtmlOtherLink+htmlOldLinks;
}
if (linkCount > 1 && archiveDownloadURL.length ()) {
@ -271,8 +300,8 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
replaceAll (allHtmlLinks, TMPL_DOWNLOAD, archiveDownloadURL);
html += allHtmlLinks;
}
html += templateHtmlFooter+"\r\n";
plain += "\r\n\r\n"+KAZ_WEB_SITE+"\r\n\r\n"+KAZ_PLAIN_WARNING+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_STOP+"\r\n";
html += templateHtmlFooter+"\n";
plain += "\n\n"+KAZ_WEB_SITE+"\n\n"+KAZ_PLAIN_WARNING+"\n"+KAZ_PLAIN_HR+"\n"+KAZ_PLAIN_STOP+"\n\n\n";
// & => &amp; done
LOG ("plain: " << plain);
LOG ("html: " << html);
@ -288,7 +317,7 @@ MainAttachment::addPrevious (const string &href, const string &name, const bool
if (oldVal.length () && name.length () && !trust)
return;
previousLinks.erase (href);
previousLinks [href] = regex_replace (name, regex (R"([\t\r\n\"]+|(\\\")|(>\s*))"), "");
previousLinks [href] = regex_replace (name, regex (R"([\t\n\"]+|(\\\")|(>\s*))"), "");
LOG ("inserted: " << href << ": " << previousLinks[href]);
}
@ -320,7 +349,7 @@ MainAttachment::extractLinks (const string &extractedPlainKAZ) {
string name;
if (startName != string::npos) {
name = string (extractedPlainKAZ, startName+3, stopName - startName - 3);
name = string (extractedPlainKAZ, startName+2, stopName -startName-2);
// skip [> \r\n\t]
string::size_type nextPos = name.find_first_not_of ("[>< \t\r\n\"]");
if (nextPos != string::npos)
@ -372,29 +401,38 @@ MainAttachment::extractLinks (const vector<string> &liOne) {
}
void
MainAttachment::extractPreviousKAZ (ifstream &mbox) {
DEF_LOG ("MainAttachment::extractPreviousKAZ", "");
string extractedPlainKAZ, extractedHtmlKAZ;
for (const Attachment *attachP : allMarkedPtrs) {
if (!(attachP->toUpdate || attachP->isKazAttachment)) // isKazAttachment => toUpdate
continue;
string textProp = attachP->getProp (contentTypeToken, textRegEx);
MainAttachment::extractPreviousKAZ (string &extractedPlainKAZ, string &extractedHtmlKAZ, ifstream &mbox, const Attachment &attach) {
DEF_LOG ("MainAttachment::extractPreviousKAZ", "attach:" << attach);
if (!(attach.toUpdate || attach.isKazAttachment)) // isKazAttachment => toUpdate
return;
string textProp = attach.getProp (contentTypeToken, textRegEx);
if (textProp.empty ())
continue;
string content (attachP->getContent (mbox));
return;
string content (attach.getContent (mbox));
replaceAll (content, "&amp;", "&");
if (textProp == PLAIN) {
LOG (PLAIN);
extractedPlainKAZ += attachP->getSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
extractedPlainKAZ += attach.getSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
}
if (textProp == HTML) {
LOG (HTML);
string section = attachP->getSection (content, KAZ_HTML_START, KAZ_HTML_STOP);
section += attachP->getSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
string section = attach.getSection (content, KAZ_HTML_START, KAZ_HTML_STOP);
section += attach.getSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
// update href from HTML attachments
replaceAll (section, "&amp;", "&");
extractedHtmlKAZ += section;
}
}
void
MainAttachment::extractPreviousKAZ (ifstream &mbox) {
DEF_LOG ("MainAttachment::extractPreviousKAZ", "");
string extractedPlainKAZ, extractedHtmlKAZ;
if (boundary.empty ())
extractPreviousKAZ (extractedPlainKAZ, extractedHtmlKAZ, mbox, *this);
else
for (const Attachment *attachP : allMarkedPtrs)
extractPreviousKAZ (extractedPlainKAZ, extractedHtmlKAZ, mbox, *attachP);
LOG ("extractedPlainKAZ: "<< extractedPlainKAZ);
extractLinks (extractedPlainKAZ);
@ -411,16 +449,54 @@ MainAttachment::extractPreviousKAZ (ifstream &mbox) {
void
MainAttachment::removePreviousArchive () {
DEF_LOG ("MainAttachment::removePreviousArchive", "");
vector<string> toRemove;
for (map <string, string>::const_iterator it = previousLinks.begin (); it != previousLinks.end (); ++it) {
const string key (it->first);
if (regex_match (key, archiveURLSignature))
if (regex_match (key, archiveURLRegex))
toRemove.push_back (key);
}
for (string old : toRemove)
previousLinks.erase (old);
}
// ================================================================================
void MainAttachment::rewriteHeaders (ifstream &mbox, ofstream &outbox, const HeaderType &headerType) {
DEF_LOG ("MainAttachment::rewriteHeaders", "headerType: " << headerType);
if (SAME == headerType) {
copy (mbox, outbox, 0, contentPos);
return;
}
string mime (getMime (mbox));
string::size_type startPos = (0);
for (string token : {string ("content-transfer-encoding"), Attachment::contentTypeToken}) {
startPos = caseInsensitiveFind (mime, token);
for (string::size_type stopPos (startPos);
(stopPos = mime.find ("\n", stopPos)) != string::npos;
) {
if (string (" \t").find (mime [stopPos+1]) == string::npos) {
mime.erase (startPos, stopPos-startPos);
break;
}
}
}
string contentType (KAZ_EMPTY_TEXT_PLAIN);
switch (headerType) {
case SAME: /* no way */;
case MAIN_PLAIN: contentType = KAZ_EMPTY_TEXT_PLAIN; break;
case ATTACH_HTML: contentType = KAZ_ATTACHMENT_TEXT_HTML; break;
case MULTI:
boundary = "__KAZ__"+boundaryGen (40);
contentType = "Content-Type: multipart/mixed; boundary=\""+boundary+"\"";
boundary = "--"+boundary+"--";
boundaryMiddleSize = boundary.length () - 2;
}
if (startPos >= mime.length ())
startPos = mime.length ()-1;
mime.insert (startPos, contentType);
outbox << mime << flush;
}
// ================================================================================
MainAttachment::MainAttachment (ifstream &mbox)
: Attachment (mbox, initTmpLevel (), 0, initTmpPos ()),
@ -440,7 +516,7 @@ MainAttachment::markSignificant (const streamoff &minAttachSize, ifstream &mbox)
bool plainMarked (false), htmlMarked (false);
markDisclaim (plainMarked, htmlMarked);
emptyEMail = ! (plainMarked || htmlMarked);
Attachment::markSignificant ("", minAttachSize, mbox, allMarkedPtrs);
Attachment::markSignificant ("", isSigned, minAttachSize, mbox, allMarkedPtrs);
}
// ================================================================================
@ -483,7 +559,7 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
int attachCount (0);
string dirName, mediaName;
for (Attachment *attachP : allMarkedPtrs) {
if (attachP->isKazAttachment || !attachP->toExtract)
if (attachP->isSigned || attachP->isKazAttachment || !attachP->toExtract)
continue;
newPjEntry (attachCount, attachP->getContentType (), attachP->getAttachName (), dirName, mediaName);
++attachCount;
@ -534,7 +610,7 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
cout << dirName << endl;
}
for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ())
if (attachP->isSigned || !attachP->embeddedData.size ())
continue;
string content = attachP->getContent (mbox);
vector<string> imgs;
@ -557,8 +633,17 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
}
// ================================================================================
/*!
Régle à appliquer dans le cas où Kaz ajoute son cartouche et que le corps principale n'est pas multipart :
<table>
<tr><th>src</th><th>FOOTER</th><th>BOTH</th><th>ATTCH</th></tr>
<tr><th>text/plain</th><td>OK</td><td>mute multi</td><td>mute multi</td></tr>
<tr><th>empty mail</th><td>mute plain</td><td>mute multi</td><td>mute html</td></tr>
</table>
*/
void
MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, const AttachMode &attachMode) {
MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, AttachMode attachMode) {
DEF_LOG ("MainAttachment::substitute", "minSize: " << minSize << " AttachMode: " << attachMode);
// preparation
@ -566,7 +651,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
removePreviousArchive ();
map<const string, const string> translateHtml;
for (Attachment *attachP : allMarkedPtrs)
if (attachP->toExtract && !attachP->isKazAttachment) {
if (!attachP->isSigned && attachP->toExtract && !attachP->isKazAttachment) {
readDownloadUrl (attachP->downloadUrl);
if (attachP->downloadUrl.empty ()) {
LOG ("no change");
@ -580,7 +665,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
}
}
for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ())
if (attachP->isSigned || !attachP->embeddedData.size ())
continue;
for (EmbeddedData &embedded : attachP->embeddedData)
readDownloadUrl (embedded.downloadUrl);
@ -590,16 +675,40 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
removePreviousArchive ();
string plainDisclaim, htmlDisclaim;
getDisclaim (plainDisclaim, htmlDisclaim);
// copy email
streamoff curPos = 0;
copy (mbox, outbox, curPos, contentPos);
curPos = contentPos;
if (plainDisclaim.size ()) {
if (emptyEMail && (attachMode & FOOTER)) {
// check no main text
HeaderType headerType (SAME);
// copy email
if (!boundary.size () && plainDisclaim.size ())
switch (attachMode) {
case NONE: LOG_BUG (true, /* */, "eMailShrinker: bug M12: nothing to do"); break;
case FOOTER: headerType = (emptyEMail ? MAIN_PLAIN : SAME); break;
case BOTH: headerType = MULTI; break;
case ATTACHMENT: headerType = ATTACH_HTML; break;
}
rewriteHeaders (mbox, outbox, headerType);
streamoff curPos = contentPos;
if (MAIN_PLAIN == headerType) {
LOG ("Replace old content with plain");
string content (plainDisclaim);
base64Encode (content);
outbox << content << endl;
outbox.flush ();
return;
}
if (ATTACH_HTML == headerType) {
LOG ("Replace old content with html");
string content (plainDisclaim);
base64Encode (content);
outbox << content << endl;
outbox.flush ();
return;
}
if (plainDisclaim.size () && emptyEMail && (attachMode & FOOTER)) {
// case : multi
LOG ("Force main text");
LOG_BUG (boundary.empty () || ! subAttachements.size (), /**/, "eMailShrinker: can't force add footer M9: : " << *this);
cerr << "eMailShrinker: force main text" << endl;
string content (plainDisclaim);
base64Encode (content);
outbox << boundary.substr (0, boundary.length () -2) << endl
@ -607,12 +716,24 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
<< content << endl;
outbox.flush ();
}
if (MULTI == headerType) {
LOG ("New boundary");
map<string, string>::const_iterator it (env.find (contentTypeToken));
LOG_BUG (it == env.end (), /* */, "eMailShrinker: bug M13: no content-type");
outbox << boundary.substr (0, boundary.length () -2) << endl
<< Attachment::contentTypeToken << ": " << it->second << endl;
}
for (Attachment *attachP : allMarkedPtrs) {
copy (mbox, outbox, curPos, attachP->beginInParent);
LOG_BUG (attachP->toUpdate && attachP->toExtract, /**/, "eMailShrinker: bug M5: update and extract. pos: " << attachP->beginPos);
if (attachP->toExtract || attachP->isKazAttachment) {
if (attachP->isSigned) {
LOG ("don't change signed content");
copy (mbox, outbox, attachP->beginInParent, attachP->endPos);
} else if (attachP->toExtract || attachP->isKazAttachment) {
LOG ("skip Extracted or previous attachments");
} else if (attachP->toUpdate) {
@ -624,6 +745,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
LOG_BUG (isPlain && isHtml, /**/, "eMailShrinker: bug M6: plain and html: " << attachP->getContentType ());
LOG_BUG (! (isPlain || isHtml), /**/, "eMailShrinker: bug M7: not plain or html: " << attachP->getContentType ());
LOG ("toUpdate: isPlain: " << isPlain << " isHtml: " << isHtml << " isDisclaimer: " << isDisclaimer);
if (attachP != this)
copy (mbox, outbox, attachP->beginInParent, attachP->contentPos);
string content = attachP->getContent (mbox);
@ -654,8 +776,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
}
removeSection (content, KAZ_HTML_START, KAZ_HTML_STOP);
removeSection (content, KAZ_PLAIN_START, KAZ_PLAIN_STOP);
// XXX case insensitive ??
if (content.find (CID) != string::npos)
if (caseInsensitiveFind (content, CID) != string::npos)
replaceAll (content, translateHtml);
attachP->replaceEmbedded (content);
}
@ -679,26 +800,31 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
}
attachP->println (outbox, content);
} else {
LOG_BUG (true, continue, "eMailShrinker: bug M8: can't change" << *attachP);
LOG_BUG (true, continue, "eMailShrinker: upload has failed, so can't change" << *attachP);
}
outbox.flush ();
curPos = attachP->endPos;
}
if (plainDisclaim.size () && (attachMode & ATTACHMENT)) {
// XXX si pas de multipart
LOG ("Add kaz attachment");
LOG_BUG (boundary.empty () || ! subAttachements.size (), /**/, "eMailShrinker: can't add Kaz attachment M10: : " << *this);
cerr << "eMailShrinker: force attachment" << endl;
if (subAttachements.size ()) {
streamoff lastPos = subAttachements.back ().endPos;
copy (mbox, outbox, curPos, lastPos);
curPos = lastPos;
}
string content (KAZ_HTML_CONTENT+htmlDisclaim+BODY_END+HTML_END);
base64Encode (content);
outbox << boundary.substr (0, boundary.length () -2) << endl
<< KAZ_ATTACHMENT_TEXT_HTML << endl
<< content << endl;
outbox.flush ();
}
copy (mbox, outbox, curPos, endPos);
if (MULTI == headerType)
outbox << boundary.substr (0, boundary.length ()) << endl;
outbox.close ();
}

View File

@ -32,7 +32,10 @@
// knowledge of the CeCILL-B license and that you accept its terms. //
////////////////////////////////////////////////////////////////////////////
#define LAST_VERSION "2.3 2022-11-25 eMailShrinker"
#include "version.hpp"
const std::string kaz::LAST_VERSION_NUM ("2.17");
const std::string kaz::LAST_VERSION_DATE ("2023-04-23");
const std::string kaz::LAST_VERSION (LAST_VERSION_NUM+" "+LAST_VERSION_DATE+" eMailShrinker");
#include <iostream>
#include <fstream>
@ -103,7 +106,7 @@ static const char *const inputFileC = inputFile.c_str ();
int
main (int argc, char** argv) {
// XXX debug before parse options
// uncomment next line in case of debug parse options
// Log::debug = true;
DEF_LOG ("main:", "");
prog = argv [0];

View File

@ -106,7 +106,7 @@ static const char *const inputFileC = inputFile.c_str ();
int
main (int argc, char** argv) {
// XXX debug before parse options
// uncomment next line in case of debug parse options
// Log::debug = true;
DEF_LOG ("main:", "");
prog = argv [0];

View File

@ -65,6 +65,14 @@ const string kaz::availableURLChars =
"abcdefghijklmnopqrstuvwxyz"
"~";
const regex kaz::encodedWordRegex ("\\s*=\\?" // flag begin
"([0-9A-Za-z!#$%&'+^_`{}~-]+)" // charset
"\\?" // flag sep
"([QqBb])" // quoted our base64
"\\?" // flag sep
"([^ ?]+)" // encoded string
"\\?=\\s*"); // flag end
// ================================================================================
uint16_t
@ -167,6 +175,19 @@ kaz::caseInsensitiveRFind (const string& s, const string& pattern, const string:
return s.rend () - it - pattern.length ();
}
string
kaz::boundaryGen (const int &size) {
static const char alphanum[] =
"0123456789"
"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
"abcdefghijklmnopqrstuvwxyz";
string result;
result.reserve (size);
for (int i = 0; i < size; ++i)
result += alphanum[rand() % (sizeof (alphanum) - 1)];
return result;
}
// ================================================================================
template<char delim>
void
@ -186,13 +207,13 @@ kaz::quotedDecode (string &content) {
}
if (p+1 < content.end () && *(p+1) == '\n') {
LOG_BUG (q == content.begin (), ++p;continue, "kazMisc::quotedDecode bug: bad quoted-printable format. (start with '=', content: " << content << ")");
LOG_BUG (q == content.begin (), ++p;continue, "kazMisc::quotedDecode bug: bad quoted-printable format. (start with '=', delim: " << int (delim) << " content: " << content << ")");
++p;
--q;
continue;
}
LOG_BUG (p+3 > content.end () || !isxdigit (p[1]) || !isxdigit (p[2]), return, "kazMisc::quotedDecode bug: bad quoted-printable format. (content: " << content << ")");
LOG_BUG (p+3 > content.end () || !isxdigit (p[1]) || !isxdigit (p[2]), return, "kazMisc::quotedDecode bug: bad quoted-printable format. (delim: " << int (delim) << " content: " << content << ")");
*q = (char) ((getHexaVal (p[1]) << 4) + getHexaVal (p[2]));
p += 2;
}
@ -259,17 +280,23 @@ kaz::base64Decode (string &content) {
LOG ("len: " << len);
unsigned char buff[4];
int idx = 0;
string::iterator p (content.begin ()), q (p);
string::iterator p (content.begin ()), q (p), lastOK (p);
for (;
p < content.end ();
++p) {
char c = *p;
if (c == '=')
break;
if (c == '\n')
if (c == '\n') {
lastOK = p;
continue;
}
LOG_BUG (!isBase64 (c), return, "kazMisc::base64Decode bug: bad base64 format. (content: " << content << ")");
if (!isBase64 (c)) {
content.resize (lastOK-content.begin ());
LOG ("kazMisc::base64Decode bug: bad base64 format. (content: " << content << ")");
return;
}
buff [idx] = getBase64Val (c);
if (++idx != 4)
continue;
@ -284,8 +311,7 @@ kaz::base64Decode (string &content) {
buff [j] = 0;
*q = buff [0] << 2 | (buff [1] & 0x30) >> 4;
++q;
--idx;
if (idx) {
if (idx > 2) {
*q = buff [1] << 4 | (buff [2] & 0x3c) >> 2;
++q;
}
@ -364,65 +390,66 @@ kaz::iso2utf (string &content) {
// ================================================================================
void
kaz::encodedWord (string &content) {
kaz::encodedWordDecode (string &content) {
// rfc2047
DEF_LOG ("kaz::extendedWord", "content: " << content);
DEF_LOG ("kaz::encodedWordDecode", "content: " << content);
string::size_type charsetPos = content.find ("=?");
if (charsetPos == string::npos)
return;
LOG ("charsetPos: " << charsetPos);
LOG_BUG (charsetPos != 0, return, "kazMisc::extendedWord bug: =? not at begin pos. (content: " << content << ")");
string result;
for ( ;
(charsetPos = content.find ("=?", charsetPos)) != string::npos;
) {
string::size_type modePos = content.find ("?", charsetPos+2);
auto pos (0);
sregex_iterator ewItEnd;
for (sregex_iterator ewIt (content.begin (), content.end (), encodedWordRegex);
ewIt != ewItEnd;
++ewIt) {
smatch m = *ewIt;
if (pos != m.position ()) {
result += content.substr (pos, m.position () - pos);
LOG ("stantad " << content.substr (pos, m.position () - pos));
}
string encoded (m[3]);
replace (encoded.begin (), encoded.end (), '_', ' ');
LOG_BUG (modePos == string::npos, return, "kazMisc::extendedWord bug: no end chartset. (content: " << content << ")");
string::size_type contentPos = content.find ("?", modePos+1);
LOG ("charset: " << m[1] << " mode: " << m[2] << " string: " << encoded);
LOG_BUG (contentPos != modePos+2, return, "kazMisc::extendedWord bug: no end chartset. (content: " << content << ")");
string::size_type endPos = content.find ("?=", contentPos+1);
LOG_BUG (endPos == string::npos, return, "kazMisc::extendedWord bug: no end chartset. (content: " << content << ")");
string tmp (content.substr (contentPos+1, endPos-contentPos-1));
switch (content [modePos+1]) {
switch (m[2].str ()[0]) {
case 'B':
case 'b':
base64Decode (tmp);
base64Decode (encoded);
break;
case 'Q':
case 'q':
quotedDecode (tmp);
quotedDecode (encoded);
break;
default:
LOG_BUG (true, return, "kazMisc::extendedWord bug: unknown mode. (mode: " << content [modePos+1] << ")");
LOG_BUG (true, return, "kazMisc::encodedWordDecode bug: unknown mode. (mode: " << m[2] << ")");
}
LOG ("tmp: " << tmp);
string charset (content.substr (charsetPos, modePos-charsetPos-2));
LOG ("decoded: " << encoded);
string charset (m[1]);
toLower (charset);
if (! caseInsensitiveFind (charset, "ISO"))
iso2utf (tmp);
result += tmp;
charsetPos = endPos+2;
iso2utf (encoded);
result += encoded;
pos = m.position () + m.str ().length ();
}
content = result;
content = result + content.substr (pos);
LOG ("content: " << content);
}
// ================================================================================
void
kaz::charsetValue (string &content) {
kaz::charsetValueDecode (string &content) {
// rfc2184
DEF_LOG ("kaz::charsetValue", "content: " << content);
DEF_LOG ("kaz::charsetValueDecode", "content: " << content);
string::size_type langPos = content.find ("'");
LOG_BUG (langPos == string::npos, return, "kazMisc::charsetValue bug: no '. (content: " << content << ")");
LOG_BUG (langPos == string::npos, return, "kazMisc::charsetValueDecode bug: no '. (content: " << content << ")");
string::size_type contentPos = content.find ("'", langPos+1);
LOG_BUG (contentPos == string::npos, return, "kazMisc::charsetValue bug: no double '. (content: " << content << ")");
LOG_BUG (contentPos == string::npos, return, "kazMisc::charsetValueDecode bug: no double '. (content: " << content << ")");
string tmp (content.substr (contentPos+1));
quotedDecode<'%'> (tmp);
LOG ("tmp: " << tmp);
@ -435,3 +462,20 @@ kaz::charsetValue (string &content) {
}
// ================================================================================
void
kaz::removeQuote (string &content) {
if (content.empty () || content [0] != '"')
return;
string::size_type stop = (1);
for (;;) {
stop = content.find ('"', stop);
if (stop == string::npos || content [stop-1] != '\\')
break;
++stop;
}
content = (stop != string::npos) ?
content.substr (1, stop-1) :
content.substr (1);
}
// ================================================================================

View File

@ -54,7 +54,7 @@ namespace kaz {
//static const vector<const string> stringsToUpdate;
static vector<string> stringsToUpdate;
/*! mime tokens */
static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, RELATED, ALTERNATIVE, KAZ_ATTACH_NAME;
static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, MULTIPART, RELATED, ALTERNATIVE, SIGNED, KAZ_ATTACH_NAME;
/*! pattern to extract mime values */
static const regex nameRegEx, nameCharsetRegEx, boundaryRegEx, cidDefRegEx, textRegEx, multiRegEx;
@ -100,7 +100,7 @@ namespace kaz {
/*! char position of attachment content */
streamoff contentPos, endPos;
/*! properties of the attachment */
bool toExtract, toUpdate, toDisclaim, isKazAttachment;
bool toExtract, toUpdate, toDisclaim, isKazAttachment, isSigned;
/*! id of an image embedded in mbox */
string cid;
/*! url to replace the attachment */
@ -131,7 +131,9 @@ namespace kaz {
/*! recursively marks alternative attachments to be disclaim */
void markDisclaim (bool &plainMarked, bool &htmlMarked);
/*! recursively marks big attachments to be removed and upated (including disclaim). return true when part need to be updated (can't be extracted). */
bool markSignificant (const string &parentMultiProp, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs);
bool markSignificant (const string &parentMultiProp, const bool &parentSigned, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs);
/*! get a copy of mime header */
string getMime (ifstream &mbox) const;
/*! get a copy of the content. Base64 is decoded. Quoted-Printable is unwarp and unquoted */
string getContent (ifstream &mbox) const;
/*! write the content, encoded if necessary (base64 and quoted-printable) */

View File

@ -51,6 +51,12 @@ namespace kaz {
ostream &operator << (ostream &out, const AttachMode &attachMode);
istream &operator >> (istream &in, AttachMode &attachMode);
enum HeaderType { SAME, MULTI, MAIN_PLAIN, ATTACH_HTML };
extern const string headerTypeLabels[];
extern const map<string, HeaderType> headerTypeMap;
ostream &operator << (ostream &out, const HeaderType &headerType);
istream &operator >> (istream &in, HeaderType &headerType);
/*! root level of e-mail structure */
class MainAttachment : public Attachment {
public:
@ -106,11 +112,16 @@ namespace kaz {
void extractLinks (const string &extractedPlainKAZ);
/*! extract previous links from html-li list. Used by extractPreviousKAZ */
void extractLinks (const vector<string> &liOne);
/*! extract previous links in mbox on one attachment section. Used by extractPreviousKAZ */
void extractPreviousKAZ (string &extractedPlainKAZ, string &extractedHtmlKAZ, ifstream &mbox, const Attachment &attach);
/*! extract previous links in mbox. Used by getUpdatedURL and substitute */
void extractPreviousKAZ (ifstream &mbox);
/*! remove previous links to archive. Used by substitute */
void removePreviousArchive ();
/*! rewrite main headers */
void rewriteHeaders (ifstream &mbox, ofstream &outbox, const HeaderType &headerType);
public:
/*! the main attachment in mbox */
MainAttachment (ifstream &mbox);
@ -124,7 +135,7 @@ namespace kaz {
/*! extract big attachments in mbox to extractDir and write to stdout le dirname of each extraction */
void extract (ifstream &mbox, const SizeArg &minSize) const;
/*! substitute big attachments by the url give in stdin */
void substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, const AttachMode &attachMode);
void substitute (ifstream &mbox, ofstream &outbox, const SizeArg &minSize, AttachMode attachMode);
};
// ================================================================================

View File

@ -38,6 +38,7 @@
#include <string>
#include <ctype.h>
#include <map>
#include <regex>
namespace kaz {
using namespace std;
@ -47,6 +48,8 @@ namespace kaz {
extern const char * const base64Chars;
/*! set of chars available in URL */
extern const string availableURLChars;
/*! pattern for encoded words */
extern const regex encodedWordRegex;
// =======================================================================
/*! get the width of the terminal */
@ -71,6 +74,9 @@ namespace kaz {
string::size_type caseInsensitiveFind (const string& s, const string& p, const string::size_type &pos = 0);
/*! reverse find upper case of p in upper case of s */
string::size_type caseInsensitiveRFind (const string& s, const string& p, const string::size_type &pos = 0);
string boundaryGen (const int &size);
/*! side effect to repplace =XX by the char with de haxe value XX. It could be %XX in rfc2184 */
template<char delim='='>
void quotedDecode (string &content);
@ -82,10 +88,12 @@ namespace kaz {
void base64Encode (string &content);
/*! side effect to change charset of content */
void iso2utf (string &content);
/*! side effect to get the encodedWord according rfc2047 */
void encodedWord (string &content);
/*! side effect to get the encoded word according rfc2047 rfc5987 rfc2978 */
void encodedWordDecode (string &content);
/*! side effect to get the charsetValue according rfc2184 */
void charsetValue (string &content);
void charsetValueDecode (string &content);
/*! side effect to remove quote */
void removeQuote (string &content);
// =======================================================================
/*! return if the c need no quote */

48
src/include/version.hpp Normal file
View File

@ -0,0 +1,48 @@
////////////////////////////////////////////////////////////////////////////
// Copyright KAZ 2021 //
// //
// contact (at) kaz.bzh //
// //
// This software is a filter to shrink email by attachment extraction. //
// //
// This software is governed by the CeCILL-B license under French law and //
// abiding by the rules of distribution of free software. You can use, //
// modify and/or redistribute the software under the terms of the //
// CeCILL-B license as circulated by CEA, CNRS and INRIA at the following //
// URL "http://www.cecill.info". //
// //
// As a counterpart to the access to the source code and rights to copy, //
// modify and redistribute granted by the license, users are provided //
// only with a limited warranty and the software's author, the holder of //
// the economic rights, and the successive licensors have only limited //
// liability. //
// //
// In this respect, the user's attention is drawn to the risks associated //
// with loading, using, modifying and/or developing or reproducing the //
// software by the user in light of its specific status of free software, //
// that may mean that it is complicated to manipulate, and that also //
// therefore means that it is reserved for developers and experienced //
// professionals having in-depth computer knowledge. Users are therefore //
// encouraged to load and test the software's suitability as regards //
// their requirements in conditions enabling the security of their //
// systems and/or data to be ensured and, more generally, to use and //
// operate it in the same conditions as regards security. //
// //
// The fact that you are presently reading this means that you have had //
// knowledge of the CeCILL-B license and that you accept its terms. //
////////////////////////////////////////////////////////////////////////////
#ifndef _kaz_version_hpp
#define _kaz_version_hpp
#include <string>
namespace kaz {
// ================================================================================
extern const std::string LAST_VERSION_NUM, LAST_VERSION_DATE, LAST_VERSION;
// ================================================================================
}
#endif // _kaz_version_hpp

View File

@ -1,5 +1,21 @@
**Dépollueur**
Le dépollueur consiste à extraire les pièces jointes pour les remplacer par des liens de téléchargement temporaire.
Le logiciel est donc très fortement lié à un logiciel de dépôt de fichier.
Le système de dépôt de proposé par Jirafeau à l'avantage de gérer les liens interne en fonction de la signature du contenu (hash).
Cela offre plusieurs avantages. les fichiers de même contenu sont factorisés sans que les utilisateurs en aient consciences.
L'intégrité des données est inhérente à ce mécanisme, puisque un lien ne va pas faire référence à un autre contenu dans le dépôt.
A partir du moment où notre dépollueur à une action sur les messages, cela à des conséquences sur lutilisation d'outils cryptographiques.
Le chiffrement de pièces jointes a peu d'incidence. L'information chiffrée se trouve stocké sur un serveur au lieu d'un ordinateur personnel (voir pire un serveur des GAFAM).
Ici, l'avantage est que l'information sera détruite au bout d'un temps spécifié.
La signature, empêche le dépollueur d'intervenir.
Dans le cas où l'on souhaite signer ses messages, il faut au préalable déposer ses pièces jointes (hors GAFAM) et faire référence dans le message aux liens de téléchargement.
Main Programmes:
* [eMailShrinker](eMailShrinker_8cpp.html)
* [jirafeauAPI](jirafeauAPI_8cpp.html)

File diff suppressed because it is too large Load Diff

View File

@ -1,32 +0,0 @@
#/bin/bash
# GME
# Ici on défini une liste de tests à effectuer sur les mails reçus (et donc dépollués !).
# Touts les mails sont concatennés avant vérification (egrep) des regexp donc on veillera,
# donc nommer différement les pièces sur les mails de tests si on a besoin de retrouver quel est le mail qui plante
# pour chaque test
TESTS_DESCRIPTION=(); # Une description pour l'affichage du résultat
TESTS_REGEXP=(); # Quoi qu'on doit trouver ?
TESTS_MATCHING_LINES=(); # Combien d'occurences de ça dans l'ensemble des messages ?
TESTS_DESCRIPTION+=("Le nombre de mails correctement dépollués");
TESTS_REGEXP+=(".*Attention : Kaz a \S+ ce message.*");
TESTS_MATCHING_LINES+=(1);
TESTS_DESCRIPTION+=("La piece est correctement nommée pour un mail normal");
TESTS_REGEXP+=("\\* \"pdftest.pdf\" <https://depot.[^/]*/");
TESTS_MATCHING_LINES+=(1);
TESTS_DESCRIPTION+=("La piece est correctement nommée pour un mail sans texte");
TESTS_REGEXP+=("\\* \"pdftestmailsanstexte.pdf\" <https://depot.[^/]*/");
TESTS_MATCHING_LINES+=(1);
TESTS_DESCRIPTION+=("La piece est correctement nommée pour une piece sans guillemets");
TESTS_REGEXP+=("\\* \"pdftestSansGuillemets.pdf\" <https://depot.[^/]*/");
TESTS_MATCHING_LINES+=(1);
TESTS_DESCRIPTION+=("L'encodage du mail est correct");
TESTS_REGEXP+=(".*é.*");
TESTS_MATCHING_LINES+=(0);

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,456 +0,0 @@
#/bin/bash
# --- GME ---
# Envois full automatisés de mails dits bizarres vers un serveur kaz,
# dans le but de tester bon fonctionnement du dépollueur
#
# On enregistre en .eml dans le dossier mails, des messages "mal construits" pouvant causer pb au dépollueur.
# Les mails ont les champs suivants à l'interieur: %DATE% %FROM_ADDRESS% %TO_ADDRESS%
# Le fichier expectedResults permet de définir des regexp qu'on doit vérifier sur le mail reçu.
#
# Le script les envoie un par un via l'utilitaire swaks
# - interne -> interne
# - interne -> externe
# - externe -> interne
#
# Un petit warning, ce script fait du mailing moche (on balance un paquet de mails volontairement mal construits).
# Vous savez ce que vous faites ! Et si vous ne savez pas, ne le faites pas.
# En particulier si on utilise des mails qui sortent et entrent ... (blacklistings / etc ...)
# On préfèrera faire les tests sur un environnement de dev bien sûr !
TEST_DIR=$(cd "$(dirname $0)"; pwd)
. $TEST_DIR/mails/expectedResults.sh
NBMAILS=$(ls $TEST_DIR/mails/*.eml | wc -l)
# random key
KEY="TESTDEP-$(echo $RANDOM | md5sum | head -c 20;)"
# ou qu'il est le depollueur ?
SERVEUR_TO_TEST=
SERVEUR_IS_LOCALHOST=0
# s'il est pas en local, il faut bien pouvoir s'identifier
USERFROM=
USERTO=
PASSWORD=
# un mail externe pour tester entrant / sortant
EXTERNAL_EMAIL=
EXTERNAL_SMTP=
EXTERNAL_USER=
EXTERNAL_PASSWORD=
EXTERNAL_SMTP_AUTH=1
EXTERNAL_TESTS_ENABLED=1
EXTERNAL_SMTP_ENABLED=1
EXTERNAL_ONLY=0
#un peu de debug
VERBOSE=0
#ou si on préfère ne rien savoir
QUIET=0
#swaks est installé ?
SWAKS_OK=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
# les résultats
SKIP_RESULTS=0
KEEP_MAILS_ON_SERVEURS=0
#mutt est installé ?
MUTT_OK=$(dpkg-query -W --showformat='${Status}\n' mutt 2>/dev/null | grep "install ok installed")
usage () {
echo "${PRG} [-h] -s <smtp_a_tester> -a <user> [-l] [-p <password>] [-v|-q] [-eq] [-ea] [-em <mail-ext>] [-es <smtp-ext>] [-eu <user-ext>] [-ep <password-ext>]"
echo " Tests full-automatisés du dépollueur."
echo " On envoie une série de mails et on vérifie que le lien généré est bon."
echo " -h | --help : Display this help."
echo " -s | --server : Quel serveur je teste ? (fqnd)"
echo " -l | --localhost : si le script est execute depuis le serveur test"
echo " -f | --from : Un compte à utiliser sur ce serveur en tant qu'expediteur"
echo " -t | --to : Un compte à utiliser sur ce serveur en tant que destinataire (optionnel, l expediteur est utilise sinon)"
echo " -p | --password : Le mot de passe de l'expéditeur si le script est executé à distance"
echo " -ed | --ext-disabled : Ne pas faire de tests externes"
echo " -eo | --ext-only : Ne faire que les tests externes"
echo " -em | --ext-mail : Un mail externe pour des tests vers / depuis l'exterieur"
echo " -es | --ext-server : Quel serveur je teste ?"
echo " -eu | --ext-user : Un identifiant pour smtp ext si différents de mail"
echo " -ep | --ext-password : Le mot de passe associé pour le ext-user ou mail"
echo " -ena | --ext-no-auth : Le smtp externe n'a pas besoin d'authentifications (FAI par exemple)"
echo " -rd | --results-disabled : Ne pas tenter d'aller chercher les résultats"
echo " -ksm | --keep-sended-mails : Laisser les messages envoyés sur les serveurs de destination"
echo " -v or -V display more informations"
echo " -q or -Q Quiet mode"
echo "Un petit warning, ce script fait du mailing moche (on balance un paquet de mails volontairement mal construits). "
echo "Vous savez ce que vous faites ! Et si vous ne savez pas, ne le faites pas."
}
cleanupPackage () {
# on vire swaks si on l'a installé
# eventuellement on le fait même silently
package=$1;
quietMode=$2;
[ -z $quietMode ] && quietMode=0;
[ $quietMode -eq 0 ] && echo "$package a été installé, vous permettez que je désinstalle .. !";
if [ $quietMode -eq 0 ]
then
apt-get -y remove $package;
apt-get -y purge $package;
else
apt-get -yq remove $package;
apt-get -yq purge $package;
fi;
}
# récupération des arguments
[ -z "$1" ] && { usage;exit; }
for arg in $*
do
case "$arg" in
'-h' | '--help' )
usage
shift
exit;;
'-s' | '--server')
shift;
SERVEUR_TO_TEST=$1
shift;;
'-l' | '--localhost')
SERVEUR_IS_LOCALHOST=1
shift;;
'-f' | '--from')
shift;
USERFROM=$1
shift;;
'-p' | '--password')
shift;
PASSWORD=$1
shift;;
'-t' | '--to')
shift;
USERTO=$1
shift;;
'-ed' | '--ext-disabled')
EXTERNAL_TESTS_ENABLED=0;
EXTERNAL_SMTP_ENABLED=0;
shift;;
'-em' | '--ext-mail')
shift;
EXTERNAL_EMAIL=$1
shift;;
'-es' | '--ext-server')
shift;
EXTERNAL_SMTP=$1
shift;;
'-eu' | '--ext-user')
shift;
EXTERNAL_USER=$1
shift;;
'-ep' | '--ext-password')
shift;
EXTERNAL_PASSWORD=$1
shift;;
'-ena' | '--ext-no-auth')
EXTERNAL_SMTP_AUTH=0
shift;;
'-eo' | '--ext-only')
EXTERNAL_ONLY=1
shift;;
'-v' | '-V' )
VERBOSE=1
shift;;
'-q' | '-Q' )
QUIET=1
shift;;
'-rd' | '--results-disabled')
SKIP_RESULTS=1
shift;;
'-ksm' | '--keep-sended-mails')
KEEP_MAILS_ON_SERVEURS=1
shift;;
*)
esac
done;
[ $VERBOSE -eq 1 -a $QUIET -eq 1 ] && echo 'Les options -q et -v ne sont pas compatibles.' && usage;
[ $VERBOSE -eq 1 ] && echo "Checking for swaks : $SWAKS_OK";
if [ -z "$SWAKS_OK" ];
then
if [ $QUIET -eq 0 ]
then
echo "Swaks doit être installé, je le fait pour vous si j'ai les droits !";
while true; do
read -p "On y va ? [Yn] :" yn
case $yn in
[YyoO]* ) break;;
[Nn]* ) echo "Swaks doit être installé, les tests en dépendent"; exit;;
* ) break;;
esac
done;
apt-get update;
apt-get -y install swaks;
else
apt-get -q update;
apt-get -qy install swaks;
fi;
SWAKS_INSTALLED=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
if [ -z "$SWAKS_INSTALLED" ]
then
echo "Swaks doit être installé, et je ne suis pas root visiblement.";
echo "Soit le faire à la main, soit je m'en charge (apt install et remove à la fin), mais il me faut les droits root !";
exit;
fi;
trap 'cleanupPackage swaks $QUIET' EXIT;
fi
if [ $SKIP_RESULTS -eq 0 -a -z "$MUTT_OK" ];
then
[ $VERBOSE -eq 1 ] && echo "Checking for mutt : $MUTT_OK";
if [ $QUIET -eq 0 ]
then
echo "Mutt doit être installé, je le fait pour vous si j'ai les droits !";
while true; do
read -p "On y va ? [Yn] :" yn
case $yn in
[YyoO]* ) break;;
[Nn]* ) echo "Mutt doit être installé, les tests en dépendent"; exit;;
* ) break;;
esac
done;
apt-get update;
apt-get -y install mutt;
else
apt-get -q update;
apt-get -qy install mutt;
fi;
MUTT_INSTALLED=$(dpkg-query -W --showformat='${Status}\n' swaks 2>/dev/null | grep "install ok installed")
if [ -z "$MUTT_INSTALLED" ]
then
echo "Mutt doit être installé, et je ne suis pas root visiblement.";
echo "Soit le faire à la main, soit je m'en charge (apt install et remove à la fin), mais il me faut les droits root !";
exit;
fi;
trap 'cleanupPackage mutt $QUIET' EXIT;
fi
# si les paramètres n'ont pas été renseignés via les arguments du script,
# on prompt ceux qui sont necessaires
[ $QUIET -eq 1 -a -z "$SERVEUR_TO_TEST" ] && SERVEUR_TO_TEST="mail.kaz.local"; #en mode quiet, on prend les options par défaut
if [ -z "$SERVEUR_TO_TEST" ]
then
[ $VERBOSE -eq 1 ] && echo "-- Le serveur n'a pas été renseigné (il faut un fqdn):";
read -p "Le serveur à tester ? [mail.kaz.local] :" SERVEUR_TO_TEST;
[ -z "$SERVEUR_TO_TEST" ] && SERVEUR_TO_TEST="mail.kaz.local";
fi;
[ $QUIET -eq 1 -a -z "$USERFROM" ] && SERVEUR_TO_TEST="contact1@kaz.local"; #en mode quiet, on prend les options par défaut
if [ -z "$USERFROM" ]
then
[ $VERBOSE -eq 1 ] && echo "-- Il me faut également un compte mail qui envoie :";
read -p "Email Expediteur ? [contact1@kaz.local]" USERFROM;
[ -z "$USERFROM" ] && USERFROM="contact1@kaz.local";
fi;
[ -z "$USERTO" ] && USERTO="$USERFROM"; #sauf si précisé en option
# si on est en local, normalement les mails partent sans besoin de renseigner l'authentification
# sinon il faut un mot de passe pour le smtp
if { [ $SKIP_RESULTS -eq 0 ] || [ $SERVEUR_IS_LOCALHOST -eq 0 ]; } && [ -z "$PASSWORD" ];
then
[ $VERBOSE -eq 1 ] && echo "-- Le password à utiliser n'a pas été renseigné :";
read -s -p "Le password ? [toto]" PASSWORD;
[ -z "$PASSWORD" ] && PASSWORD="toto";
fi;
# utilisation d'un mail exterieur et eventuellement un smtp exterieur
if [ -z "$EXTERNAL_EMAIL" -a $EXTERNAL_TESTS_ENABLED -eq 1 -a $QUIET -eq 0 ]
then
[ $VERBOSE -eq 1 ] && echo "-- Les tests vers/depuis l'exterieur n'ont pas été desactivé via l'option -ed.";
while true; do
read -p "On fait des tests depuis / vers l'exterieur aussi ? [Yn] :" yn
case $yn in
[YyoO]* ) break;;
[Nn]* ) EXTERNAL_TESTS_ENABLED=0; EXTERNAL_SMTP_ENABLED=0; break;;
* ) break;;
esac
done;
if [ $EXTERNAL_TESTS_ENABLED -eq 0 ]
then
[ $VERBOSE -eq 1 ] && echo "-- Les tests vers/depuis l'exterieur annulés";
else
[ $VERBOSE -eq 1 ] && echo "-- Ok mais je n'ai toujours pas l'adresse exterieure à utiliser.";
read -p "Mail externe ? :" EXTERNAL_EMAIL;
fi
fi
if [ -z "$EXTERNAL_SMTP" -a $EXTERNAL_TESTS_ENABLED -eq 1 -a $QUIET -eq 0 ]
then
[ $VERBOSE -eq 1 ] && echo "-- smtp, je pourrais le deviner à partir du mail mais ... j'ai la flemme là.";
echo "Pour les tests Depuis l'exterieur, il me faut aussi un smtp.Laisser vide si pas de tests depuis l'exterieur."
read -p "smtp externe ? :" EXTERNAL_SMTP;
[ -z $EXTERNAL_SMTP ] && EXTERNAL_SMTP_ENABLED=0;
fi
# authent pour le smtp ext
if [ -z "$EXTERNAL_USER" -a $EXTERNAL_SMTP_AUTH -eq 1 -a $EXTERNAL_SMTP_ENABLED -eq 1 -a $QUIET -eq 0 ]
then
[ $VERBOSE -eq 1 ] && echo "L'authentification au smtp externe n'a pas été désactivée avec l'option -ea :";
[ $VERBOSE -eq 1 ] && echo "Le user à utiliser pour ce smtp externe n'a pas été renseigné :";
read -p "Le user du smtp ? " EXTERNAL_USER;
fi
if [ -z "$EXTERNAL_PASSWORD" -a $EXTERNAL_SMTP_AUTH -eq 1 -a $EXTERNAL_SMTP_ENABLED -eq 1 -a $QUIET -eq 0 ]
then
[ $VERBOSE -eq 1 ] && echo "L'authentification au smtp externe n'a pas été désactivée avec l'option -ea :";
[ $VERBOSE -eq 1 ] && echo "Le password à utiliser pour ce smtp externe n'a pas été renseigné :";
read -s -p "Le password pour le smtp ? " EXTERNAL_PASSWORD;
fi
####
#### Fin de la récupération des paramètres du script. On balance les mails !
####
if [ $VERBOSE -eq 1 ]
then
echo 'Paramètres : '
echo '------------'
echo " Serveur : $SERVEUR_TO_TEST"
echo " Expéditeur interne : $USERFROM"
echo " Destinataire interne : $USERTO"
echo ' Password : ***'
echo " Tests vers externes : $EXTERNAL_TESTS_ENABLED"
echo " Tests depuis externe : $EXTERNAL_SMTP_ENABLED"
echo " Mail externe : $EXTERNAL_EMAIL"
echo " Auth externe : $EXTERNAL_USER"
echo ''
echo ''
fi
if [ $QUIET -eq 0 ]
then
sleep 2;
echo "OK on va y aller ... mais j'attend encore 3 secondes pour donner le temps pour un Ctrl C"
sleep 3;
fi
if [ $VERBOSE -eq 1 ]
then
echo "C'est parti !";
# vous êtes sûr ? bon d'accord c'est chiant,
# mais si on a cree des mails vraiment veroles, et qu'on est en train de les balancer sur la en prod, on peut avoir un doute ...
fi
sleep 2;
if [ $EXTERNAL_ONLY -eq 0 ]
then
echo "1 ----- $NBMAILS Mails envoyés depuis $USERFROM vers $USERTO ----";
for MAIL in "$TEST_DIR/mails/"*.eml
do
echo "$MAIL";
options=(-f $USERFROM -t $USERTO -d $MAIL --h-From: "\"$KEY\" <$USERFROM>");
[ $SERVEUR_IS_LOCALHOST -eq 0 ] && options+=(-s $SERVEUR_TO_TEST -p 587 --tlso -au $USERFROM -ap $PASSWORD);
[ $QUIET -eq 1 ] && options+=(--silent);
[ $VERBOSE -eq 0 ] && options+=(-n);
swaks "${options[@]}";
done
fi
if [ $EXTERNAL_TESTS_ENABLED -eq 1 ]
then
echo "2 ----- $NBMAILS Mails envoyés depuis $USERFROM vers $EXTERNAL_USER ----";
for MAIL in "$TEST_DIR/mails/"*.eml
do
options=(-f $USERFROM -t $EXTERNAL_EMAIL -d $MAIL -s $SERVEUR_TO_TEST --tlso --h-From: "\"$KEY\" <$USERFROM>");
[ $SERVEUR_IS_LOCALHOST -eq 0 ] && options+=(-p 587 -au $USERFROM -ap $PASSWORD); # on rajoute l'authetification
[ $QUIET -eq 1 ] && options+=(--silent);
[ $VERBOSE -eq 0 ] && options+=(-n);
swaks "${options[@]}";
done
fi
if [ $EXTERNAL_SMTP_ENABLED -eq 1 ]
then
if [ $SERVEUR_IS_LOCALHOST -eq 1 ]
then
[ $QUIET -eq 0 ] && echo "3 ----- Mails envoyés depuis l'extérieur vers localhost ... heu une autre fois en fait ... ";
[ $VERBOSE -eq 1 ] && echo "-- Mais si vous renseignez un fqdn pour le serveur je pourrais essayer ...";
else
echo "3 ----- $NBMAILS Mails envoyés depuis $EXTERNAL_EMAIL vers $USERTO ----";
for MAIL in "$TEST_DIR/mails/"*.eml
do
options=(-f $EXTERNAL_EMAIL -t $USERTO -d $MAIL -s $EXTERNAL_SMTP -p 587 --tlso --h-From: "\"$KEY\" <$USERFROM>");
[ $EXTERNAL_SMTP_AUTH -eq 1 ] && options+=(-au $EXTERNAL_USER -ap $EXTERNAL_PASSWORD);
[ $QUIET -eq 1 ] && options+=(--silent);
[ $VERBOSE -eq 0 ] && options+=(-n);
swaks "${options[@]}";
done
fi
fi
####
#### Et bien maintenant on va voir les résultats !
####
if [ $SKIP_RESULTS -eq 0 ]
then
if [ $VERBOSE -eq 1 ]
then
echo "-- On va maintenant chercher les mails en imap pour checker que tout est bon !";
echo "-- Creation d'un repertoire temporaire /tmp/$KEY";
echo "-- Fichier de conf pour mutt /tmp/$KEY/muttrc";
fi
sleep 5;
[ $VERBOSE -eq 1 ] && echo "-- OK on va y aller ... mais j'attend encore 3 secondes pour donner le temps aux messages d'être traités par les serveurs ...";
sleep 3;
mkdir /tmp/$KEY
echo "set imap_user=$USERFROM
set ssl_force_tls = yes
set ssl_starttls = yes
set imap_pass=$PASSWORD
set folder=imaps://$SERVEUR_TO_TEST:993
set spoolfile = +INBOX
set pipe_decode=yes
set wait_key=no
mailboxes +INBOX
" > /tmp/$KEY/muttrc
[ $VERBOSE -eq 1 ] && echo "-- on télécharge les mails qu'on colle dans un txt";
mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><tag-pattern>all<enter><tag-prefix><pipe-message>cat >> /tmp/$KEY/mails.eml<enter><quit>'"
[ $VERBOSE -eq 1 ] && echo "-- Et on fait le ménage sur le serveur";
if [ $KEEP_MAILS_ON_SERVEURS -eq 1 ]
then
[ $VERBOSE -eq 1 ] && echo "-- On les enregistre dans un sous dossier $KEY";
mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><tag-pattern>all<enter><tag-prefix><decode-save><enter><enter><quit>y'"
else
[ $VERBOSE -eq 1 ] && echo "-- On supprime les mails identifiés par la clef";
mutt -F /tmp/$KEY/muttrc -e "push '<limit>~f $KEY<enter><delete-pattern>all<enter><enter><quit>y'"
fi
NBTESTS=${#TESTS_DESCRIPTION[@]};
RED='\033[0;31m';
GREEN='\033[0;32m'; # Green
NC='\033[0m'; # No Color
[ $VERBOSE -eq 1 ] && echo "-- On passe les $NBTESTS tests définis dans le repertoire de mails";
echo "-------- RESULTATS DES TESTS ------ ";
for (( i=0; i<${NBTESTS}; i++ ));
do
RESULTCOUNT=$(egrep -c "${TESTS_REGEXP[$i]}" /tmp/$KEY/mails.eml);
[ -z $RESULTCOUNT ] && RESULTCOUNT=0;
if [ $RESULTCOUNT -eq ${TESTS_MATCHING_LINES[$i]} ]
then
echo -e "--$i-- ${TESTS_DESCRIPTION[$i]} : ${GREEN}OK${NC}";
else
echo -e "--$i-- ${TESTS_DESCRIPTION[$i]} : ${RED}KO${NC}";
fi
done
fi