enh: collector_mail_attach Decrypt GPG attachments

Author: e3rd

.github/workflows/unittests.yml

@@ -72,7 +72,8 @@ jobs:
     - name: Run basic testsuite
       if: ${{ matrix.type == 'basic' }}
       run: pytest --cov intelmq/ --cov-report=xml --cov-branch intelmq/
-
+    - name: Assure permissions
+      run: chmod -R u=rwX,go= intelmq/tests/bots/collectors/mail/gpg_ring/
     - name: Run full testsuite
       if: ${{ matrix.type == 'full' }}
       run: pytest --cov intelmq/ --cov-report=xml --cov-branch intelmq/ contrib/
@@ -81,3 +82,4 @@ jobs:
         INTELMQ_TEST_DATABASES: 1
         INTELMQ_TEST_EXOTIC: 1
         INTELMQ_TEST_INSTALLATION: 1
+        GPG_RING_PATH: intelmq/tests/bots/collectors/mail/gpg_ring/

CHANGELOG.md

@@ -24,6 +24,7 @@ Please refer to the [NEWS](NEWS.md) for a list of changes which have an affect o
 
 ### Bots
 #### Collectors
+- `intelmq.bots.collectors.mail.collector_mail_attach`: Decrypt GPG attachments (PR#2623 by Edvard Rejthar).
 
 #### Parsers
 - `intelmq.bots.parsers.cymru.parser_cap_program`: Add mapping for TOR and ipv6-icmp protocol (PR#2621 by Mikk Margus Möll).

intelmq/bots/collectors/mail/REQUIREMENTS.txt

@@ -2,3 +2,4 @@
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
 imbox>=0.8.5
+python-gnupg>=0.5

intelmq/bots/collectors/mail/collector_mail_attach.py

@@ -10,15 +10,18 @@
 
 Uses the common mail iteration method from the lib file.
 """
+from functools import cache, cached_property
 import re
+
 from intelmq.lib.utils import unzip
-from intelmq.lib.exceptions import InvalidArgument
+from intelmq.lib.exceptions import InvalidArgument, MissingDependencyError
 
 from ._lib import MailCollectorBot
 
 
 class MailAttachCollectorBot(MailCollectorBot):
     """Monitor IMAP mailboxes and retrieve mail attachments"""
+
     attach_regex: str = "csv.zip"
     extract_files: bool = True
     folder: str = "INBOX"
@@ -28,11 +31,16 @@ class MailAttachCollectorBot(MailCollectorBot):
     mail_user: str = "<user>"
     rate_limit: int = 60
     subject_regex: str = "<subject>"
+    decrypt: bool = False  # Decrypt the attachment with GPG
+
+    gpg_passphrase: str = ""  # The private key passhrase
+
+    gpg_home: str = ""  # Change the GPG home directory.
 
     def init(self):
         super().init()
         if self.attach_regex is None:
-            raise InvalidArgument('attach_regex', expected='string')
+            raise InvalidArgument("attach_regex", expected="string")
 
     def process_message(self, uid, message):
         seen = False
@@ -42,12 +50,14 @@ def process_message(self, uid, message):
                 continue
 
             try:
-                attach_filename = attach['filename']
+                attach_filename = attach["filename"]
             except KeyError:
                 # https://github.com/certtools/intelmq/issues/1538
-                self.logger.debug('Skipping attachment because of missing filename.')
+                self.logger.debug("Skipping attachment because of missing filename.")
                 continue
-            if attach_filename.startswith('"'):  # for imbox versions older than 0.9.5, see also above
+            if attach_filename.startswith(
+                '"'
+            ):  # for imbox versions older than 0.9.5, see also above
                 attach_filename = attach_filename[1:-1]
 
             if re.search(self.attach_regex, attach_filename):
@@ -57,18 +67,33 @@ def process_message(self, uid, message):
                 report = self.new_report()
 
                 if self.extract_files:
-                    raw_reports = unzip(attach['content'].read(), self.extract_files,
-                                        return_names=True, logger=self.logger)
+                    raw_reports = unzip(
+                        attach["content"].read(),
+                        self.extract_files,
+                        return_names=True,
+                        logger=self.logger,
+                    )
                 else:
-                    raw_reports = ((attach_filename, attach['content'].read()), )
+                    raw_reports = ((attach_filename, attach["content"].read()),)
 
                 for file_name, raw_report in raw_reports:
+                    if self.decrypt:
+                        gpg = self._gpg().decrypt(
+                            raw_report, passphrase=self.gpg_passphrase
+                        )
+                        if gpg.ok:
+                            raw_report = gpg.data
+                        else:
+                            self.logger.error('Could not decrypt attachment %s: %s.', file_name, gpg.status)
+                            continue
                     report = self.new_report()
                     report.add("raw", raw_report)
                     if file_name:
                         report.add("extra.file_name", file_name)
                     report["extra.email_subject"] = message.subject
-                    report["extra.email_from"] = ','.join(x['email'] for x in message.sent_from)
+                    report["extra.email_from"] = ",".join(
+                        x["email"] for x in message.sent_from
+                    )
                     report["extra.email_message_id"] = message.message_id
                     report["extra.email_date"] = message.date
                     self.send_message(report)
@@ -80,5 +105,13 @@ def process_message(self, uid, message):
         self.logger.info("Email report read.")
         return seen
 
+    @cache
+    def _gpg(self):
+        try:
+            from gnupg import GPG
+        except ImportError:
+            raise MissingDependencyError("python-gnupg", ">=0.5")
+        return GPG(gnupghome=self.gpg_home)
+
 
 BOT = MailAttachCollectorBot

intelmq/tests/bots/collectors/mail/gpg_attachment.eml

@@ -0,0 +1,40 @@
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="===============0256050913907025376=="
+Subject: foobar zip
+From: Sebastian Wagner <wagner@cert.at>
+To: cert@example.com
+Message-ID: <07ce0153-060b-f48d-73d9-d92a20b3b3aa@cert.at>
+Date: Tue, 3 Sep 2019 16:57:40 +0200
+Content-Language: en-US
+
+--===============0256050913907025376==
+Content-Type: text/html; charset="utf-8"
+Content-Transfer-Encoding: 7bit
+
+<html>
+  <head>
+
+    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
+  </head>
+  <body text="#000000" bgcolor="#FFFFFF">
+    Please look at the attachment<br>
+  </body>
+</html>
+
+--===============0256050913907025376==
+Content-Type: application/octet-stream
+Content-Transfer-Encoding: base64
+Content-Disposition: attachment; filename="foobar.txt.gpg"
+MIME-Version: 1.0
+
+hQGMA9ig68HPFWOpAQv/fsRXK1mdmGw83CdbIOZeUlFVx2KKEONWVYIu/CHPxqxREYUWTva+XxYE
+LirKR9xK9lG19H6BSSZD3xXMahAShRzgs1dHH4UMxGqdpiXo5DbdVkTS7kg5+hG7QX20x+ExwsRd
+r76/gzY+Lmsv4hydxMHyG/w0OBsV/0mLWGAAVzer9Y8xkUP6jB16XzoSQGkfCP+7DpK224135JgU
+E6XSD2p+WKzALxXiET4IfinJnG8sSTHlkCQZgBp8lpDPdP2BbHo0KmvAofxaRqkvMfpVk7kUyy2W
+8tENKSEPu0sKEX8HZhT8Sw3X2pn8ggbAadHRjOzOO58MUvWJvwURqa1LIITA4pmyhw4svp1UzODb
+Q2Cd+MyWikjD4CebJ7P/JpnoDokux9qxWKnuuvmM4mqr6bvL+Sztud4V2Z60idokbzuEaJR6ZztT
+SBMatdh/OHcUnCaYCJiEOlrgk0hYcZ12XR5C8OBAI4St9F5eGjMB0l1tZz7CeQ6MrAoPY2pZ54XO
+0k0BA3PpG+K/khPOns8rW2mlYKgWfJqRdc91EasA4+iFoM+iFwyDQGBKbAdCu0BeIf9j7t5s5LOG
+iPWI5m2AIWf+aFNqKotGFAWXeSbMWw==
+
+--===============0256050913907025376==--

intelmq/tests/bots/collectors/mail/gpg_ring/crls.d/DIR.txt

@@ -0,0 +1 @@
+v:1:

intelmq/tests/bots/collectors/mail/gpg_ring/openpgp-revocs.d/3C8124A8245618D286CF871E94CE2905DB00CDB7.rev

@@ -0,0 +1,35 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub   rsa3072 2019-12-11 [S]
+      3C8124A8245618D286CF871E94CE2905DB00CDB7
+uid          Example identity 2 <envelope-example-identity-2@example.com>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used.  It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key.  However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation.  For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below.  Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQG2BCABCgAgFiEEPIEkqCRWGNKGz4celM4pBdsAzbcFAl3xFwwCHQAACgkQlM4p
+BdsAzbeofgv+JUQ4Qb40Z0p2ML+jciJjNiRdKYOUyhY1UKECwq6QlbW/MSkQMn4g
+esJfevFwGHKxaI4bw9ywFMtR/UB91YDY4D+lURm4qMuc8pFYEBSMhro0x8ToWz1E
+vupwAqTF484ybBrujg2UaOox9BbyhIbsiAH3ttB6wThCbXdhkxp/w6qUaWo+n5Ra
+5xWrtFkHJx+WIE4mzxqvnuN3RtA9tkr3L8oavw8Vy0j44lhkVE4Mrl/XDGOnryvv
+3WYuiHjOdTP5plb/p4veTvCfUZOcrmhTiwU8gQWMQfy3Nr1NmIhNwoXRTwB41ogt
+j3/mijdKVlRkzzmungIt0G3NJM8sAI9TiZ6KyaH4g3fVHU0ddfUwd5sCV3Q+UHfJ
+iUZeO1qHA8PA0NZwquRE2rnGdc63nHE/7AD9SjpvqIoBcvfoZCniCBm70pSmJvSP
+Csd2/TvaI8PQ28vAVY2LKqCX6JX1MvkiJOZbDGl+VBhCavQCQ8lR0d2b+wriDa3y
+2rhCXk0+ozc+
+=92kE
+-----END PGP PUBLIC KEY BLOCK-----

intelmq/tests/bots/collectors/mail/gpg_ring/openpgp-revocs.d/F14F2E8097E0CCDE93C4E871F4A4F26779FA03BB.rev

@@ -0,0 +1,35 @@
+This is a revocation certificate for the OpenPGP key:
+
+pub   rsa3072 2019-12-11 [S]
+      F14F2E8097E0CCDE93C4E871F4A4F26779FA03BB
+uid          Example identity <envelope-example-identity@example.com>
+
+A revocation certificate is a kind of "kill switch" to publicly
+declare that a key shall not anymore be used.  It is not possible
+to retract such a revocation certificate once it has been published.
+
+Use it to revoke this key in case of a compromise or loss of
+the secret key.  However, if the secret key is still accessible,
+it is better to generate a new revocation certificate and give
+a reason for the revocation.  For details see the description of
+of the gpg command "--generate-revocation" in the GnuPG manual.
+
+To avoid an accidental use of this file, a colon has been inserted
+before the 5 dashes below.  Remove this colon with a text editor
+before importing and publishing this revocation certificate.
+
+:-----BEGIN PGP PUBLIC KEY BLOCK-----
+Comment: This is a revocation certificate
+
+iQG2BCABCgAgFiEE8U8ugJfgzN6TxOhx9KTyZ3n6A7sFAl3xFtwCHQAACgkQ9KTy
+Z3n6A7vCwwv/U1mKrClA3lxfI+lTAecMeYwlYtYcAveDjJkMs3AtAUQyqd6B7I5F
+RXQL50xjiB+S+yv3WoXd4eQbT9Z2g1OnMJfSHzENM0K+yosUJS+TpN3SU7YxJ9GR
+4J3eDxo0IyB0mL1QFQXcsWSu92yAyQpwJnscg6S0hvxiq8ij+OyDPNctDtpxcArr
+orh9+rGfLBNABemtKgVgzmedjyB7fdPYjgCi/xJL8eYUxlRUgwuwCS7A3DBaSdwU
+bd4dGvGTvzSmKp8OrbW1j7yf4Vq91qVcSqAdv1y0EpKwnnefAPJV8TsFr4wg0GpI
+ocKDGMiQMPekxfeDPrZcII4jjYXlk6WRzXPTQySZcHBN6o/TKBjdanZv5iV4JktU
+jIvru6M7JAM0PJPFj+AwY6hLV3p741qCYYEUlFs9Yjq7j4vhoHqgdVbmZSVcditn
+KLcM6F7jEx0odcbL157/xLHHqsCPVoZrr9uZVuN8J6uIYw0GTXFRlpMhDEpVg+Jd
+mB4WSiNPkql0
+=rwt1
+-----END PGP PUBLIC KEY BLOCK-----