fix PGP (skip signed message)

This commit is contained in:
François 2022-12-23 11:39:44 +01:00
parent 694570a454
commit 596ae82fe4
7 changed files with 87 additions and 36 deletions

View File

@ -33,7 +33,7 @@ depollueur/
## Compilation ## Compilation
```bash ```bash
sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev doxygen sudo apt-get install --fix-missing build-essential make g++ libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libcurl4-gnutls-dev libssl-dev doxygen dos2unix
git clone https://git.kaz.bzh/KAZ/depollueur.git git clone https://git.kaz.bzh/KAZ/depollueur.git
# or for contributors : # or for contributors :

3
src/bash/filterTest.sh Executable file → Normal file
View File

@ -33,6 +33,7 @@ usage () {
######################################## ########################################
mbox=$(realpath $1) mbox=$(realpath $1)
dos2unix "${mbox}"
cd $(dirname $0) cd $(dirname $0)
DOMAINNAME="$(cat domainname)" DOMAINNAME="$(cat domainname)"
JIRAFEAU_URL="https://depot.${DOMAINNAME}" JIRAFEAU_URL="https://depot.${DOMAINNAME}"
@ -126,7 +127,7 @@ cat "${TMP_DIR}/PJ-name.txt" | {
esac esac
} }
else else
LOG " - ${GREEN} no archive${NC}" LOG " - ${GREEN}no archive${NC}"
echo "arch: none" echo "arch: none"
fi fi
} > "${TMP_DIR}/PJ-Keys.txt" } > "${TMP_DIR}/PJ-Keys.txt"

View File

@ -60,8 +60,10 @@ const string Attachment::contentIDToken ("content-id");
const string Attachment::PLAIN ("plain"); const string Attachment::PLAIN ("plain");
const string Attachment::HTML ("html"); const string Attachment::HTML ("html");
const string Attachment::RELATED ("related"); const string Attachment::RELATED ("related");
const string Attachment::SIGNED ("signed");
const string Attachment::ALTERNATIVE ("alternative"); const string Attachment::ALTERNATIVE ("alternative");
const string Attachment::KAZ_ATTACH_NAME (".---KazAttachment---.html"); const string Attachment::KAZ_ATTACH_NAME (".---KazAttachment---.html");
const string Attachment::MULTIPART ("multipart/");
const regex Attachment::nameCharsetRegEx (".*name\\*=[ \t]*(.*)"); const regex Attachment::nameCharsetRegEx (".*name\\*=[ \t]*(.*)");
@ -76,7 +78,7 @@ const regex Attachment::nameRegEx (".*name=[ \t]*((\"(\\\\.|[^\\\\\r])*\")|[^\
const regex Attachment::boundaryRegEx (".*boundary=[ \t]*((\"(\\\\.|[^\\\\])*\")|[^; ]*);?.*"); const regex Attachment::boundaryRegEx (".*boundary=[ \t]*((\"(\\\\.|[^\\\\])*\")|[^; ]*);?.*");
const regex Attachment::cidDefRegEx (".*<([^>]*)>.*"); const regex Attachment::cidDefRegEx (".*<([^>]*)>.*");
const regex Attachment::textRegEx (".*text/("+PLAIN+"|"+HTML+").*"); const regex Attachment::textRegEx (".*text/("+PLAIN+"|"+HTML+").*");
const regex Attachment::multiRegEx ("\\s*multipart/(mixed|"+RELATED+"|"+ALTERNATIVE+").*"); const regex Attachment::multiRegEx ("\\s*"+MULTIPART+"(mixed|"+RELATED+"|"+ALTERNATIVE+"|"+SIGNED+").*");
const string Attachment::IMG_BEGIN ("<IMG"); const string Attachment::IMG_BEGIN ("<IMG");
const string Attachment::IMG_END (">"); const string Attachment::IMG_END (">");
@ -85,7 +87,6 @@ const string Attachment::IMG_END (">");
static const string SRC_BEGIN ("SRC=\""); static const string SRC_BEGIN ("SRC=\"");
static const string RFC822 ("message/rfc822"); static const string RFC822 ("message/rfc822");
static const string MULTIPART ("multipart/");
// ================================================================================ // ================================================================================
string string
@ -270,6 +271,7 @@ Attachment::Attachment (ifstream &mbox, const int &level, const streamoff beginI
toUpdate (false), toUpdate (false),
toDisclaim (false), toDisclaim (false),
isKazAttachment (false), isKazAttachment (false),
isSigned (false),
boundaryMiddleSize (0) { boundaryMiddleSize (0) {
DEF_LOG ("Attachment::Attachment", "curPos: " << curPos << " level: " << level); DEF_LOG ("Attachment::Attachment", "curPos: " << curPos << " level: " << level);
readMime (mbox, curPos); readMime (mbox, curPos);
@ -321,6 +323,9 @@ Attachment::readMime (ifstream &mbox, streamoff &curPos) {
cid = getProp (contentIDToken, cidDefRegEx); cid = getProp (contentIDToken, cidDefRegEx);
if (caseInsensitiveFind (getContentType (), MULTIPART) != string::npos) { if (caseInsensitiveFind (getContentType (), MULTIPART) != string::npos) {
string multiProp = getProp (contentTypeToken, multiRegEx);
if (SIGNED == multiProp)
isSigned = true;
boundary = getProp (contentTypeToken, boundaryRegEx); boundary = getProp (contentTypeToken, boundaryRegEx);
removeQuote (boundary); removeQuote (boundary);
LOG ("boundary: " << boundary); LOG ("boundary: " << boundary);
@ -416,18 +421,19 @@ Attachment::markDisclaim (bool &plainMarked, bool &htmlMarked) {
// ================================================================================ // ================================================================================
bool bool
Attachment::markSignificant (const string &parentMultiProp, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs) { Attachment::markSignificant (const string &parentMultiProp, const bool &parentSigned, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs) {
DEF_LOG ("Attachment::markSignificant", "parentMultiProp: " << parentMultiProp << " minAttachSize: " << minAttachSize); DEF_LOG ("Attachment::markSignificant", "parentMultiProp: " << parentMultiProp << " minAttachSize: " << minAttachSize);
isSigned |= parentSigned;
string textProp = getProp (contentTypeToken, textRegEx); string textProp = getProp (contentTypeToken, textRegEx);
bool cantBeExtract ((parentMultiProp == ALTERNATIVE && (textProp == PLAIN || textProp == HTML)) || bool cantBeExtract ((parentMultiProp == ALTERNATIVE && (textProp == PLAIN || textProp == HTML)) ||
(parentMultiProp == RELATED && textProp == HTML)); (parentMultiProp == RELATED && textProp == HTML));
string multiProp = getProp (contentTypeToken, multiRegEx); string multiProp = getProp (contentTypeToken, multiRegEx);
for (Attachment &sub : subAttachements) for (Attachment &sub : subAttachements)
cantBeExtract |= sub.markSignificant (multiProp, minAttachSize, mbox, allMarkedPtrs); cantBeExtract |= sub.markSignificant (multiProp, parentSigned || isSigned, minAttachSize, mbox, allMarkedPtrs);
if (getProp (contentTypeToken, textRegEx) == HTML) { if (getProp (contentTypeToken, textRegEx) == HTML) {
if (KAZ_ATTACH_NAME == getAttachName ()) { if (KAZ_ATTACH_NAME == getAttachName ())
isKazAttachment = true; isKazAttachment = true;
} else { else {
string content = getContent (mbox); string content = getContent (mbox);
vector<string> imgs; vector<string> imgs;
getSection (content, IMG_BEGIN, IMG_END, imgs); getSection (content, IMG_BEGIN, IMG_END, imgs);
@ -438,16 +444,27 @@ Attachment::markSignificant (const string &parentMultiProp, const streamoff &min
} }
cantBeExtract |= toUpdate; cantBeExtract |= toUpdate;
if (boundary.empty () && getSize () >= minAttachSize && !cantBeExtract) if (boundary.empty () && getSize () >= minAttachSize && !cantBeExtract)
cantBeExtract = toExtract = true; // XXX cantBeExtract ? cantBeExtract = toExtract = true;
if (toExtract || toUpdate || toDisclaim || isKazAttachment) if (toExtract || toUpdate || toDisclaim || isKazAttachment)
allMarkedPtrs.push_back (this); allMarkedPtrs.push_back (this);
return cantBeExtract; return cantBeExtract;
} }
// ================================================================================
string
Attachment::getMime (ifstream &mbox) const {
DEF_LOG ("Attachment::getMime", "beginPos: " << beginPos << " contentPos: " << contentPos);
string mime;
mime.resize (contentPos-beginPos);
mbox.seekg (beginPos, ios::beg);
mbox.read (&mime[0], contentPos-beginPos);
return mime;
}
// ================================================================================ // ================================================================================
string string
Attachment::getContent (ifstream &mbox) const { Attachment::getContent (ifstream &mbox) const {
DEF_LOG ("Attachment::getContent", "contentPos: " << contentPos); DEF_LOG ("Attachment::getContent", "contentPos: " << contentPos << " endPos: " << endPos);
string content; string content;
content.resize (endPos-contentPos); content.resize (endPos-contentPos);
mbox.seekg (contentPos, ios::beg); mbox.seekg (contentPos, ios::beg);
@ -505,7 +522,9 @@ Attachment::replaceEmbedded (string &content) const {
ostream& ostream&
kaz::operator << (ostream& os, const Attachment& attachment) { kaz::operator << (ostream& os, const Attachment& attachment) {
string prop, sep; string prop, sep;
if (attachment.toExtract) { prop = "to extract"; sep = ", "; } if (attachment.isSigned) { prop += sep+"signed"; sep = ", "; }
if (attachment.isKazAttachment) { prop += sep+"kazDisclaim"; sep = ", "; }
if (attachment.toExtract) { prop += sep+"to extract"; sep = ", "; }
if (attachment.toUpdate) { prop += sep+"need update"; sep = ", "; } if (attachment.toUpdate) { prop += sep+"need update"; sep = ", "; }
if (attachment.toDisclaim) { prop += sep+"need diclaim"; sep = ", "; } if (attachment.toDisclaim) { prop += sep+"need diclaim"; sep = ", "; }
if (attachment.embeddedData.size ()) { prop += sep+"embeddedData"; } if (attachment.embeddedData.size ()) { prop += sep+"embeddedData"; }

View File

@ -177,8 +177,8 @@ MainAttachment::readArchiveUrl () {
archiveDownloadURL.clear (); archiveDownloadURL.clear ();
string line; string line;
getline (cin, line); getline (cin, line);
LOG_BUG (line.rfind ("arch: ", 0) != 0, return, "eMailShrinker: bug 9: no archive link. (line: " << line << ")"); LOG_BUG (line.rfind ("arch: ", 0) != 0, return, "eMailShrinker: bug M9: no archive link. (line: " << line << ")");
LOG_BUG (line.rfind ("arch: bad", 0) == 0, return, "eMailShrinker: bug 10: bad archive link. (line: " << line << ")"); LOG_BUG (line.rfind ("arch: bad", 0) == 0, return, "eMailShrinker: bug M10: bad archive link. (line: " << line << ")");
if (line.rfind ("arch: none", 0) == 0) if (line.rfind ("arch: none", 0) == 0)
return; return;
archiveDownloadURL = line.substr (6); archiveDownloadURL = line.substr (6);
@ -192,7 +192,7 @@ MainAttachment::readDownloadUrl (string &url) {
string line; string line;
getline (cin, line); getline (cin, line);
LOG ("get URL: " << line); LOG ("get URL: " << line);
LOG_BUG (line.rfind ("url: ", 0) != 0, return, "eMailShrinker: bug 11: no download link. (line: " << line << ")"); LOG_BUG (line.rfind ("url: ", 0) != 0, return, "eMailShrinker: bug M11: no download link. (line: " << line << ")");
url = line.substr (5); url = line.substr (5);
} }
@ -231,7 +231,7 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
int linkCount (0); int linkCount (0);
string plainNewLinks, htmlNewLinks; string plainNewLinks, htmlNewLinks;
for (Attachment *attachP : allMarkedPtrs) { for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->toExtract) if (attachP->isSigned || !attachP->toExtract)
continue; continue;
addLink (plainNewLinks, htmlNewLinks, attachP->downloadUrl, attachP->getAttachName ()); addLink (plainNewLinks, htmlNewLinks, attachP->downloadUrl, attachP->getAttachName ());
++linkCount; ++linkCount;
@ -240,7 +240,7 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
// previousLinks.erase (attachP->downloadUrl); // previousLinks.erase (attachP->downloadUrl);
} }
for (Attachment *attachP : allMarkedPtrs) { for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ()) if (attachP->isSigned || !attachP->embeddedData.size ())
continue; continue;
for (EmbeddedData &embedded : attachP->embeddedData) { for (EmbeddedData &embedded : attachP->embeddedData) {
addLink (plainNewLinks, htmlNewLinks, embedded.downloadUrl, embedded.name); addLink (plainNewLinks, htmlNewLinks, embedded.downloadUrl, embedded.name);
@ -449,7 +449,7 @@ MainAttachment::markSignificant (const streamoff &minAttachSize, ifstream &mbox)
bool plainMarked (false), htmlMarked (false); bool plainMarked (false), htmlMarked (false);
markDisclaim (plainMarked, htmlMarked); markDisclaim (plainMarked, htmlMarked);
emptyEMail = ! (plainMarked || htmlMarked); emptyEMail = ! (plainMarked || htmlMarked);
Attachment::markSignificant ("", minAttachSize, mbox, allMarkedPtrs); Attachment::markSignificant ("", isSigned, minAttachSize, mbox, allMarkedPtrs);
} }
// ================================================================================ // ================================================================================
@ -492,7 +492,7 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
int attachCount (0); int attachCount (0);
string dirName, mediaName; string dirName, mediaName;
for (Attachment *attachP : allMarkedPtrs) { for (Attachment *attachP : allMarkedPtrs) {
if (attachP->isKazAttachment || !attachP->toExtract) if (attachP->isSigned || attachP->isKazAttachment || !attachP->toExtract)
continue; continue;
newPjEntry (attachCount, attachP->getContentType (), attachP->getAttachName (), dirName, mediaName); newPjEntry (attachCount, attachP->getContentType (), attachP->getAttachName (), dirName, mediaName);
++attachCount; ++attachCount;
@ -543,7 +543,7 @@ MainAttachment::extract (ifstream &mbox, const SizeArg &minSize) const {
cout << dirName << endl; cout << dirName << endl;
} }
for (Attachment *attachP : allMarkedPtrs) { for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ()) if (attachP->isSigned || !attachP->embeddedData.size ())
continue; continue;
string content = attachP->getContent (mbox); string content = attachP->getContent (mbox);
vector<string> imgs; vector<string> imgs;
@ -575,7 +575,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
removePreviousArchive (); removePreviousArchive ();
map<const string, const string> translateHtml; map<const string, const string> translateHtml;
for (Attachment *attachP : allMarkedPtrs) for (Attachment *attachP : allMarkedPtrs)
if (attachP->toExtract && !attachP->isKazAttachment) { if (!attachP->isSigned && attachP->toExtract && !attachP->isKazAttachment) {
readDownloadUrl (attachP->downloadUrl); readDownloadUrl (attachP->downloadUrl);
if (attachP->downloadUrl.empty ()) { if (attachP->downloadUrl.empty ()) {
LOG ("no change"); LOG ("no change");
@ -589,7 +589,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
} }
} }
for (Attachment *attachP : allMarkedPtrs) { for (Attachment *attachP : allMarkedPtrs) {
if (!attachP->embeddedData.size ()) if (attachP->isSigned || !attachP->embeddedData.size ())
continue; continue;
for (EmbeddedData &embedded : attachP->embeddedData) for (EmbeddedData &embedded : attachP->embeddedData)
readDownloadUrl (embedded.downloadUrl); readDownloadUrl (embedded.downloadUrl);
@ -601,11 +601,32 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
getDisclaim (plainDisclaim, htmlDisclaim); getDisclaim (plainDisclaim, htmlDisclaim);
// copy email // copy email
streamoff curPos = 0; if (plainDisclaim.size () && emptyEMail && boundary.empty ()) {
if (boundary.empty () && plainDisclaim.size () && (attachMode & ATTACHMENT)) { // only one attachment must be replace
// XXX if no multipart ? cerr << "eMailShrinker: force one attachment" << endl;
LOG_BUG (true, /* */, "eMailShrinker: bug 12: not multipart."); string mime (getMime (mbox));
string::size_type startPos = (0);
for (string token : {string ("Content-Transfer-Encoding"), Attachment::contentTypeToken}) {
startPos = caseInsensitiveFind (mime, "Content-Transfer-Encoding");
for (string::size_type stopPos (startPos);
(stopPos = mime.find ("\n", stopPos)) != string::npos;
) {
if (string (" \t").find (mime [stopPos+1]) == string::npos) {
mime.erase (startPos, stopPos-startPos);
break;
}
}
}
mime.insert (startPos, KAZ_EMPTY_TEXT_PLAIN);
string content (plainDisclaim);
base64Encode (content);
outbox << mime
<< content << endl;
outbox.flush ();
outbox.close ();
return;
} }
streamoff curPos = 0;
copy (mbox, outbox, curPos, contentPos); copy (mbox, outbox, curPos, contentPos);
curPos = contentPos; curPos = contentPos;
@ -626,9 +647,12 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
copy (mbox, outbox, curPos, attachP->beginInParent); copy (mbox, outbox, curPos, attachP->beginInParent);
LOG_BUG (attachP->toUpdate && attachP->toExtract, /**/, "eMailShrinker: bug M5: update and extract. pos: " << attachP->beginPos); LOG_BUG (attachP->toUpdate && attachP->toExtract, /**/, "eMailShrinker: bug M5: update and extract. pos: " << attachP->beginPos);
if (attachP->toExtract || attachP->isKazAttachment) { if (attachP->isSigned) {
LOG ("skip Extracted or previous attachments"); LOG ("don't change signed content");
copy (mbox, outbox, attachP->beginInParent, attachP->endPos);
} else if (attachP->toExtract || attachP->isKazAttachment) {
LOG ("skip Extracted or previous attachments");
} else if (attachP->toUpdate) { } else if (attachP->toUpdate) {
string textProp = attachP->getProp (contentTypeToken, textRegEx); string textProp = attachP->getProp (contentTypeToken, textRegEx);
@ -718,7 +742,7 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
outbox.flush (); outbox.flush ();
} }
copy (mbox, outbox, curPos, endPos); copy (mbox, outbox, curPos, endPos);
outbox << endl; //outbox << endl;
outbox.close (); outbox.close ();
} }

View File

@ -33,8 +33,8 @@
//////////////////////////////////////////////////////////////////////////// ////////////////////////////////////////////////////////////////////////////
#include "version.hpp" #include "version.hpp"
const std::string kaz::LAST_VERSION_NUM ("2.7"); const std::string kaz::LAST_VERSION_NUM ("2.8");
const std::string kaz::LAST_VERSION_DATE ("2022-12-19"); const std::string kaz::LAST_VERSION_DATE ("2022-12-23");
const std::string kaz::LAST_VERSION (LAST_VERSION_NUM+" "+LAST_VERSION_DATE+" eMailShrinker"); const std::string kaz::LAST_VERSION (LAST_VERSION_NUM+" "+LAST_VERSION_DATE+" eMailShrinker");
#include <iostream> #include <iostream>

View File

@ -259,17 +259,22 @@ kaz::base64Decode (string &content) {
LOG ("len: " << len); LOG ("len: " << len);
unsigned char buff[4]; unsigned char buff[4];
int idx = 0; int idx = 0;
string::iterator p (content.begin ()), q (p); string::iterator p (content.begin ()), q (p), lastOK (p);
for (; for (;
p < content.end (); p < content.end ();
++p) { ++p) {
char c = *p; char c = *p;
if (c == '=') if (c == '=')
break; break;
if (c == '\n') if (c == '\n') {
lastOK = p;
continue; continue;
}
LOG_BUG (!isBase64 (c), return, "kazMisc::base64Decode bug: bad base64 format. (content: " << content << ")"); if (isBase64 (c)) {
content.resize (lastOK-content.begin ());
LOG ("kazMisc::base64Decode bug: bad base64 format. (content: " << content << ")");
}
buff [idx] = getBase64Val (c); buff [idx] = getBase64Val (c);
if (++idx != 4) if (++idx != 4)
continue; continue;

View File

@ -54,7 +54,7 @@ namespace kaz {
//static const vector<const string> stringsToUpdate; //static const vector<const string> stringsToUpdate;
static vector<string> stringsToUpdate; static vector<string> stringsToUpdate;
/*! mime tokens */ /*! mime tokens */
static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, RELATED, ALTERNATIVE, KAZ_ATTACH_NAME; static const string contentTypeToken, contentDispositionToken, contentTransferEncodingToken, base64Token, quotedPrintableToken, contentIDToken, PLAIN, HTML, MULTIPART, RELATED, ALTERNATIVE, SIGNED, KAZ_ATTACH_NAME;
/*! pattern to extract mime values */ /*! pattern to extract mime values */
static const regex nameRegEx, nameCharsetRegEx, boundaryRegEx, cidDefRegEx, textRegEx, multiRegEx; static const regex nameRegEx, nameCharsetRegEx, boundaryRegEx, cidDefRegEx, textRegEx, multiRegEx;
@ -100,7 +100,7 @@ namespace kaz {
/*! char position of attachment content */ /*! char position of attachment content */
streamoff contentPos, endPos; streamoff contentPos, endPos;
/*! properties of the attachment */ /*! properties of the attachment */
bool toExtract, toUpdate, toDisclaim, isKazAttachment; bool toExtract, toUpdate, toDisclaim, isKazAttachment, isSigned;
/*! id of an image embedded in mbox */ /*! id of an image embedded in mbox */
string cid; string cid;
/*! url to replace the attachment */ /*! url to replace the attachment */
@ -131,7 +131,9 @@ namespace kaz {
/*! recursively marks alternative attachments to be disclaim */ /*! recursively marks alternative attachments to be disclaim */
void markDisclaim (bool &plainMarked, bool &htmlMarked); void markDisclaim (bool &plainMarked, bool &htmlMarked);
/*! recursively marks big attachments to be removed and upated (including disclaim). return true when part need to be updated (can't be extracted). */ /*! recursively marks big attachments to be removed and upated (including disclaim). return true when part need to be updated (can't be extracted). */
bool markSignificant (const string &parentMultiProp, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs); bool markSignificant (const string &parentMultiProp, const bool &parentSigned, const streamoff &minAttachSize, ifstream &mbox, vector<Attachment *> &allMarkedPtrs);
/*! get a copy of mime header */
string getMime (ifstream &mbox) const;
/*! get a copy of the content. Base64 is decoded. Quoted-Printable is unwarp and unquoted */ /*! get a copy of the content. Base64 is decoded. Quoted-Printable is unwarp and unquoted */
string getContent (ifstream &mbox) const; string getContent (ifstream &mbox) const;
/*! write the content, encoded if necessary (base64 and quoted-printable) */ /*! write the content, encoded if necessary (base64 and quoted-printable) */