KAZ/depollueur
3
1
Fork 0
Code Issues Pull Requests Projects Releases Activity

fix synchro plain and html

Browse Source
This commit is contained in:
François 2022-10-31 07:08:19 +01:00
parent 99e888f5d1
commit b4ea2185f0
7 changed files with 94 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -41,3 +41,14 @@ git clone https://git.kaz.bzh/KAZ/depollueur.git
make -j $(nproc)
```
##Astuces
Pour la mise au point dans Kaz, il faut bloquer la mise à jour automatique avec :
``` bash
touch /kaz/git/depollueur/dontupdage
/kaz/bin/container.sh stop jirafeau postfix; docker system prune; /kaz/bin/install.sh jirafeau postfix
```

26
src/Jirafeau/a.php
View File

@ -12,6 +12,7 @@
a.php?s=mel@domain.org&t=password + [action] => manage account
action: a=login a=logout a=r[on|off] a=p[minute|hour|day|week|month|quarter]
*/
use PHPMailer\PHPMailer\PHPMailer;
use PHPMailer\PHPMailer\SMTP;
use PHPMailer\PHPMailer\Exception;
@ -748,10 +749,17 @@ if ($doDownload) {
$modif = true;
continue;
}
$srcName = $dstName = ($link ['file_name'] ? $link ['file_name'] : M_NO_FILENAME);
if (in_array ($srcName, $singleName))
for ($i = 0; $i < 10000; ++$i) {
$dstName = sprintf ("%s-%2d", $srcName, $i);
$dstName = ($link ['file_name'] ? $link ['file_name'] : M_NO_FILENAME);
if (in_array ($dstName, $singleName)) {
$dstFilename = $dstName;
$dstExtension = "";
$dstExtensionPos = strrpos ($dstName, '.');
if ($dstExtensionPos) {
$dstFilename = substr ($dstName, 0, $dstExtensionPos);
$dstExtension = substr ($dstName, $dstExtensionPos);
}
for ($i = 1; $i < 10000; ++$i) {
$dstName = sprintf ("%s-%02d%s", $dstFilename, $i, $dstExtension);
if (!in_array ($dstName, $singleName)) {
if (isset ($archiveContent [T_RENAME]))
++$archiveContent [T_RENAME];
@ -761,6 +769,7 @@ if ($doDownload) {
break;
}
}
}
$singleName [] = $dstName;
$archiveContent [$cat][T_ENTRIES][] = [T_HASH => $link ['hash'], T_FILENAME => $dstName, T_CRYPT_KEY => $cryptKey, T_CRYPTED => $link ['crypted']];
}
@ -1080,6 +1089,15 @@ echo
'<select name="'.A_ACTION.'" style="width: auto !important;">';
foreach ($periodText as $item => $text)
echo ' <option value="'.A_PERIOD.$item.'"'.$defaultChecked [$item].'>'.$text.'</option>';
echo
'</select> '.
'<button type="submit">'."valider".'</button>'.
'</form>'.
'<form method="post">'.
'Je préfère envoyer mes messages en langue '.
'<select name="'.A_ACTION.'" style="width: auto !important;">';
foreach ($langText as $item => $text)
echo ' <option value="'.A_LANG.$item.'"'.$defaultChecked [$item].'>'.$text.'</option>';
echo
'</select> '.
'<button type="submit">'."valider".'</button>'.

20
src/bash/filter.sh
View File

@ -66,6 +66,7 @@ ARCHIVE_TITLE="archive_content"
ARCHIVE_MIME="text/kaz_email_archive"
KEEP_FAILED=true
DEBUG=
#################### FONCTIONS ############################################
BOLD=''
@ -134,7 +135,7 @@ ARCHIVE_CONTENT="${REP_PIECE_JOINTE}/archive-content.txt"
JIRAFEAU_ERROR="${REP_PIECE_JOINTE}/jirafeau-error.txt"
# Clean up when done or when aborting.
trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
[ -z "${DEBUG}" ] && trap "rm -rf in.$$ in.$$.altered ${REP_PIECE_JOINTE}" 0 1 2 3 15
if ! cat > "in.$$"; then
LOG_FIC "${RED}Cannot save mail to file${NC}"
@ -142,8 +143,7 @@ if ! cat > "in.$$"; then
fi
LOG_FIC "\n" \
" size: ${YELLOW}$(wc -c < "${INSPECT_DIR}/in.$$")${NC}"
# XXX trace
# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.bak"
[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$" "${DIR_LOG}/pb/in.$$.orig")
mkdir -p "${REP_PIECE_JOINTE}/"
>"${OLD_LINKS}"
@ -156,8 +156,7 @@ echo "time: ${DATE_TEMPS}\nid: $(date +%s)" > "${ARCHIVE_CONTENT}"
LOG_FIC "${CYAN}${SHRINK_CMD} -u \"${INSPECT_DIR}/in.$$\" 2>> \"${FIC_LOG}\" > \"${OLD_LINKS}\"${NC}"
"${SHRINK_CMD}" -u "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" > "${OLD_LINKS}"
cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK
do
cat "${OLD_LINKS}" | grep "${JIRAFEAU_URL}" | while read REMOTE_LINK; do
REMOTE_REF=$(echo "${REMOTE_LINK}" | sed -e 's/.*h=\([^&]*\).*/\1/' -e 's/.*http.*//')
[ -z "${REMOTE_REF}" ] && continue
REMOTE_KEY=$(echo "${REMOTE_LINK}" | grep "k=" | sed 's%.*k=\([^&]*\).*%\1%')
@ -171,10 +170,8 @@ LOG_FIC " - archive starts with: ${NL}${YELLOW}$(cat ${ARCHIVE_CONTENT})${NC}"
# Etape extraction des pieces jointes
LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${INSPECT_DIR}/in.$$ ${NC}"
"${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" -d "${REP_PIECE_JOINTE}" "${INSPECT_DIR}/in.$$" 2>> "${FIC_LOG}" | {
while read ATTACH_TMP_NAME
do
if [ -d "${ATTACH_TMP_NAME}" ]
then
while read ATTACH_TMP_NAME; do
if [ -d "${ATTACH_TMP_NAME}" ]; then
ATTACH_MEDIA="${ATTACH_TMP_NAME}/media"
ATTACH_NAME=$(grep "^Name: " "${ATTACH_TMP_NAME}/meta" | cut -c 7- )
ATTACH_CONTENT_TYPE=$(grep "^Content-Type: " "${ATTACH_TMP_NAME}/meta" | cut -c 15- )
@ -195,7 +192,7 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
case "${JIR_TOKEN}" in
"" | no | *Error* | \<* )
LOG_FIC " - ${RED}Can't upload <${ATTACH_NAME}> <${ATTACH_CONTENT_TYPE}> <$(wc -c < "${ATTACH_MEDIA}")> (${JIR_TOKEN}) <in.$$.bak>. It will be not change in e-mail.${NC}"
echo "new:"
echo "url:"
keepFailed "${INSPECT_DIR}/in.$$"
echo "UPLOAD_FAIL" >> "${JIRAFEAU_ERROR}"
;;
@ -243,8 +240,7 @@ LOG_FIC "${CYAN}${SHRINK_CMD} -s ${MAX_KEEP_IN_MAIL} -d ${REP_PIECE_JOINTE} ${IN
LOG_FIC "${CYAN}${SHRINK_CMD} -s \"${MAX_KEEP_IN_MAIL}\" \"${INSPECT_DIR}/in.$$\" \"${INSPECT_DIR}/in.$$.altered\" 2>> \"${FIC_LOG}\"${NC}"
} | "${SHRINK_CMD}" -s "${MAX_KEEP_IN_MAIL}" "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" 2>> "${FIC_LOG}"
# XXX trace
# cp "${INSPECT_DIR}/in.$$" "${INSPECT_DIR}/in.$$.altered" /var/mail/tmp/
[ -n "${DEBUG}" ] && (mkdir -p "${DIR_LOG}/pb/" ; cp "${INSPECT_DIR}/in.$$.altered" "${DIR_LOG}/pb/in.$$.altered")
if [ -s "${JIRAFEAU_ERROR}" ]; then
LOG_FIC " - ${RED}upload fail${NC}"

65
src/cpp/MainAttachment.cpp
View File

@ -32,14 +32,15 @@
// knowledge of the CeCILL-B license and that you accept its terms. //
////////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <vector>
#include <set>
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <iostream>
#include <math.h>
#include <algorithm>
#include <regex>
#include <set>
#include <unistd.h>
#include <vector>
#include "kazDebug.hpp"
#include "kazMisc.hpp"
@ -55,6 +56,10 @@ static const string TMPL_DOWNLOAD = "{{DOWNLOAD}}";
static const string TMPL_FILENAME = "{{FILENAME}}";
static const string CID = "cid:";
// "l=/" => v1 compatibility
static const regex archiveURLSignature (".*(([&?]g=)|([&?]l=/)).*");
static const string KAZ_PLAIN_HR = "______________________________________________________________________________";
static const string KAZ_PLAIN_START = "~~ PJ-KAZ !"; // don't end whith space
static const string KAZ_PLAIN_STOP = KAZ_PLAIN_START+" ~~";
@ -93,8 +98,8 @@ vector <string>
Attachment::stringsToUpdate ({KAZ_PLAIN_START, "\""+CID});
// ================================================================================
const string MainAttachment::templatePlainAddLink (" - "+TMPL_DOWNLOAD+" "+TMPL_FILENAME+"\r\n");
const string MainAttachment::templatePlainAllLink ("\r\n"+KAZ_PLAIN_DOWLOAD_ALL+"\r\n * "+TMPL_DOWNLOAD+"\r\n");
const string MainAttachment::templatePlainAddLink (" * "+TMPL_FILENAME+" <"+TMPL_DOWNLOAD+">\r\n");
const string MainAttachment::templatePlainAllLink ("\r\n * "+KAZ_PLAIN_DOWLOAD_ALL+" <"+TMPL_DOWNLOAD+">\r\n");
const string MainAttachment::templateHtmlHeader (KAZ_HTML_START+"<p style=\"clear: left; padding: 1pc 0 0 0; font-size:10px; color:#969696;\">"+KAZ_PLAIN_START+"</p><hr>\n"
"<div class=\"kaz\">"
@ -167,6 +172,7 @@ MainAttachment::addLink (string &plain, string &html, const string &url, const s
plain += plainNewOneLink;
string htmlNewOneLink (templateHtmlAddLink);
string codedUrl (url);
// XXX amp ?
replaceAll (codedUrl, "&", "&amp;");
replaceAll (htmlNewOneLink, TMPL_DOWNLOAD, codedUrl);
replaceAll (htmlNewOneLink, TMPL_FILENAME, name);
@ -218,9 +224,9 @@ MainAttachment::getDisclaim (string &plain, string &html) const {
string allPlainLinks (templatePlainAllLink);
replaceAll (allPlainLinks, TMPL_DOWNLOAD, archiveDownloadURL);
plain += allPlainLinks;
string allLinks (templateHtmlAllLink);
replaceAll (allLinks, TMPL_DOWNLOAD, archiveDownloadURL);
html += allLinks;
string allHtmlLinks (templateHtmlAllLink);
replaceAll (allHtmlLinks, TMPL_DOWNLOAD, archiveDownloadURL);
html += allHtmlLinks;
}
html += templateHtmlFooter;
plain += "\r\n\r\n"+KAZ_WEB_SITE+"\r\n"+KAZ_PLAIN_HR+"\r\n"+KAZ_PLAIN_STOP+"\r\n";
@ -245,6 +251,7 @@ MainAttachment::addPrevious (const string &href, const string &name, const bool
void
MainAttachment::extractLinks (const string &extractedPlainKAZ) {
// plain text => "* name <url>"
DEF_LOG ("Attachment::extractedPlainKAZ", "extractedPlainKAZ: " << extractedPlainKAZ);
for (string::size_type startPos (0);
(startPos = extractedPlainKAZ.find ("http", startPos)) != string::npos;
@ -257,22 +264,30 @@ MainAttachment::extractLinks (const string &extractedPlainKAZ) {
if (extractedPlainKAZ [stopPos])
++stopPos;
streamoff stopName = startPos;
startPos = stopPos;
// get all href but KAZ_WEB_SITE
// the archive link will be skip by filter.sh
if (href == KAZ_WEB_SITE)
continue;
string::size_type nextPos = extractedPlainKAZ.find ("http", startPos);
string name (extractedPlainKAZ, startPos, (nextPos == string::npos ? extractedPlainKAZ.length () : nextPos) - startPos);
// skip [> \r\n\t]
nextPos = name.find_first_not_of ("[> \t\r\n\"]");
if (nextPos != string::npos)
name.erase (0, nextPos);
// stop before [>\r\n\t]
nextPos = name.find_first_of ("[\"]");
if (nextPos != string::npos)
name.erase (nextPos);
// backward find name
string::size_type startName = extractedPlainKAZ.rfind ("* ", stopName);
string name;
if (startName != string::npos) {
name = string (extractedPlainKAZ, startName+3, stopName - startName - 3);
// skip [> \r\n\t]
string::size_type nextPos = name.find_first_not_of ("[>< \t\r\n\"]");
if (nextPos != string::npos)
name.erase (0, nextPos);
// stop before [>\r\n\t]
nextPos = name.find_first_of ("[\"<]");
if (nextPos != string::npos)
name.erase (nextPos);
}
LOG ("plain name: " << name);
addPrevious (href, name);
}
@ -281,6 +296,7 @@ MainAttachment::extractLinks (const string &extractedPlainKAZ) {
// ================================================================================
void
MainAttachment::extractLinks (const vector<string> &liOne) {
// html text => "<li ...><a href="url">name</a>"
DEF_LOG ("Attachment::extractedPlainKAZ", "liOne.size: " << liOne.size ());
for (const string &one : liOne) {
if (caseInsensitiveFind (one, CLASS_ONE) == string::npos)
@ -353,10 +369,11 @@ MainAttachment::extractPreviousKAZ (ifstream &mbox) {
void
MainAttachment::removePreviousArchive () {
vector<string> toRemove;
for (map <string, string>::const_iterator it = previousLinks.begin (); it != previousLinks.end (); ++it)
if (it->first.find ("&g=") != string::npos ||
it->first.find ("&l=/") != string::npos) // v1 compatibility
toRemove.push_back (it->first);
for (map <string, string>::const_iterator it = previousLinks.begin (); it != previousLinks.end (); ++it) {
const string key (it->first);
if (regex_match (key, archiveURLSignature))
toRemove.push_back (key);
}
for (string old : toRemove)
previousLinks.erase (old);
}
@ -521,8 +538,10 @@ MainAttachment::substitute (ifstream &mbox, ofstream &outbox, const SizeArg &min
continue;
for (EmbeddedData &embedded : attachP->embeddedData)
readDownloadUrl (embedded.downloadUrl);
// XXX test empty ?
}
readArchiveUrl ();
removePreviousArchive ();
string plainDisclaim, htmlDisclaim;
getDisclaim (plainDisclaim, htmlDisclaim);
// copy email

10
src/cpp/eMailShrinker.cpp
View File

@ -32,7 +32,7 @@
// knowledge of the CeCILL-B license and that you accept its terms. //
////////////////////////////////////////////////////////////////////////////
#define LAST_VERSION "2.0 2022-02-08 eMailShrinker"
#define LAST_VERSION "2.1 2022-10-30 eMailShrinker"
#include <iostream>
#include <fstream>
@ -188,10 +188,10 @@ main (int argc, char** argv) {
MainAttachment attachment (mbox);
mbox.close ();
if (attachment.getBoundary ().empty ()) {
cerr << "no attachment" << endl;
return 1;
}
// if (attachment.getBoundary ().empty ()) {
// cerr << "no attachment" << endl;
// return 1;
// }
// parse structure
mbox.open (inputName);
attachment.markSignificant (minAttachSize, mbox);

7
src/cpp/jirafeauAPI.cpp
View File

@ -32,7 +32,7 @@
// knowledge of the CeCILL-B license and that you accept its terms. //
////////////////////////////////////////////////////////////////////////////
#define LAST_VERSION "1.0 2021-02-21 jirafeauAPI"
#define LAST_VERSION "1.1 2022-10-30 jirafeauAPI"
#include <iostream>
#include <string>
@ -281,12 +281,13 @@ main (int argc, char** argv) {
}
curl_easy_setopt (easyhandle, CURLOPT_MIMEPOST, multipart);
curl_easy_perform (easyhandle);
CURLcode res (curl_easy_perform (easyhandle));
curl_easy_cleanup (easyhandle);
cout << readBuffer << endl;
showTime ("Upload");
if (res != CURLE_OK)
cerr << prog << " failed: " << curl_easy_strerror (res) << endl;
return 0;
}

4
src/include/Attachment.hpp
View File

@ -35,9 +35,9 @@
#ifndef _kaz_Attachment_hpp
#define _kaz_Attachment_hpp
#include <string>
#include <regex>
#include <map>
#include <regex>
#include <string>
#include <utility>
#include "EmbeddedData.hpp"