diff --git a/.github/workflows/test_config_syntax.yml b/.github/workflows/test_config_syntax.yml new file mode 100644 index 0000000..38ad6f9 --- /dev/null +++ b/.github/workflows/test_config_syntax.yml @@ -0,0 +1,20 @@ +name: Validate configuration syntax +on: [push, workflow_dispatch] +jobs: + test-syntax: + runs-on: ubuntu-latest + strategy: + matrix: + logstash-version: ['8.14.1', '7.17.22'] + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Validate configuration syntax + env: + LOGSTASH_VERSION: ${{matrix.logstash-version}} + run: | + docker run --rm \ + --volume ./postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume ./50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ + logstash:${LOGSTASH_VERSION} \ + logstash --config.test_and_exit -f /usr/share/logstash/pipeline/50-filter-postfix.conf diff --git a/.github/workflows/run_tests.yml b/.github/workflows/test_grok_patterns.yml similarity index 60% rename from .github/workflows/run_tests.yml rename to .github/workflows/test_grok_patterns.yml index db028b5..160f8ec 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/test_grok_patterns.yml @@ -1,14 +1,15 @@ -on: [push] +name: Test grok patterns +on: [push, workflow_dispatch] jobs: - build: + test-patterns: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: submodules: true - uses: ruby/setup-ruby@v1 with: - ruby-version: 3.1 + ruby-version: '3.2' - run: gem install jls-grok minitest - run: ruby test/test.rb diff --git a/50-filter-postfix.conf b/50-filter-postfix.conf index ee183d9..b48acdb 100644 --- a/50-filter-postfix.conf +++ b/50-filter-postfix.conf @@ -175,6 +175,13 @@ filter { tag_on_failure => [ "_grok_postfix_script_nomatch" ] add_tag => [ "_grok_postfix_success" ] } + } else if [program] =~ /^postfix.*\/verify$/ { + grok { + patterns_dir => "/etc/logstash/patterns.d" + match => [ "message", "^%{POSTFIX_VERIFY}$" ] + tag_on_failure => [ "_grok_postfix_verify_nomatch" ] + add_tag => [ "_grok_postfix_success" ] + } } else if [program] =~ /^postfix.*/ { mutate { add_tag => [ "_grok_postfix_program_nomatch" ] @@ -194,7 +201,7 @@ filter { if [postfix_client] { grok { patterns_dir => "/etc/logstash/patterns.d" - match => ["postfix_client", "^%{POSTFIX_CLIENT_INFO}$"] + match => ["postfix_client", "^%{POSTFIX_CLIENT}$"] tag_on_failure => [ "_grok_kv_postfix_client_nomatch" ] remove_field => [ "postfix_client" ] } @@ -202,7 +209,7 @@ filter { if [postfix_relay] { grok { patterns_dir => "/etc/logstash/patterns.d" - match => ["postfix_relay", "^%{POSTFIX_RELAY_INFO}$"] + match => ["postfix_relay", "^%{POSTFIX_RELAY}$"] tag_on_failure => [ "_grok_kv_postfix_relay_nomatch" ] remove_field => [ "postfix_relay" ] } @@ -237,6 +244,8 @@ filter { "postfix_client_port", "integer", "postfix_cmd_auth", "integer", "postfix_cmd_auth_accepted", "integer", + "postfix_cmd_bdat", "integer", + "postfix_cmd_bdat_accepted", "integer", "postfix_cmd_count", "integer", "postfix_cmd_count_accepted", "integer", "postfix_cmd_data", "integer", @@ -247,6 +256,8 @@ filter { "postfix_cmd_helo_accepted", "integer", "postfix_cmd_mail", "integer", "postfix_cmd_mail_accepted", "integer", + "postfix_cmd_noop", "integer", + "postfix_cmd_noop_accepted", "integer", "postfix_cmd_quit", "integer", "postfix_cmd_quit_accepted", "integer", "postfix_cmd_rcpt", "integer", @@ -266,6 +277,9 @@ filter { "postfix_size", "integer", "postfix_status_code", "integer", "postfix_termination_signal", "integer", + "postfix_tls_server_signature_size", "integer", + "postfix_verify_cache_dropped", "integer", + "postfix_verify_cache_retained", "integer", # list of float fields "postfix_delay", "float", diff --git a/51-filter-postfix-aggregate.conf b/51-filter-postfix-aggregate.conf deleted file mode 100644 index c46b895..0000000 --- a/51-filter-postfix-aggregate.conf +++ /dev/null @@ -1,38 +0,0 @@ -filter { - if ![postfix_queueid] { - drop {} - } else if [program] == "postfix/qmgr" and [postfix_from]{ - aggregate { - task_id => "%{postfix_queueid}" - code => " - map['postfix_from'] = event.get('postfix_from') - map['postfix_size'] = event.get('postfix_size') - map['postfix_nrcpt'] = event.get('postfix_nrcpt') - " - } - } else if [program] == "postfix/smtpd" { - aggregate { - task_id => "%{postfix_queueid}" - code => " - map['postfix_client_hostname'] = event.get('postfix_client_hostname') - map['postfix_client_ip'] = event.get('postfix_client_ip') - " - } - } else if [program] == "postfix/cleanup" { - aggregate { - task_id => "%{postfix_queueid}" - code => " - map['postfix_message-id'] = event.get('postfix_message-id') - " - } - } else if [program] == "postfix/smtp" { - aggregate { - task_id => "%{postfix_queueid}" - code => " - map.each do |key, value| - event.set(key, value) - end - " - } - } -} diff --git a/ALTERNATIVE-INPUTS.md b/ALTERNATIVE-INPUTS.md index 071b852..6c2973b 100644 --- a/ALTERNATIVE-INPUTS.md +++ b/ALTERNATIVE-INPUTS.md @@ -7,9 +7,11 @@ Various other Logstash filters can produce the needed input fields (`program` an ---------------- ``` -grok { - match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" } - overwrite => ["timestamp", "message"] +filter { + grok { + match => { "message" => "%{SYSLOGTIMESTAMP:timestamp} %{SYSLOGHOST} %{DATA:program}(?:\[%{POSINT}\])?: %{GREEDYDATA:message}" } + overwrite => ["timestamp", "message"] + } } ``` diff --git a/CHANGES.md b/CHANGES.md index ff7eb10..321910e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,10 +1,3 @@ # Changelog -This file contains a list of changes for the postfix-grok-patterns project. - -## 1.0.0 - -- Switched testing setup from Travis-CI to Github Actions. -- Allowed `NOQUEUE` to be parsed as a queue id too, making it easier to parse logging for e-mail that was not accepted by Postfix. -- Added a Changelog -- Decided to try and add a way to version the project content. +See [Github releases](https://github.com/whyscream/postfix-grok-patterns/releases) for all changes. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..bd92d80 --- /dev/null +++ b/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2014-2023, Tom Hendrikx + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 7ef073b..4e3eed9 100644 --- a/README.md +++ b/README.md @@ -19,31 +19,22 @@ The included Logstash config file requires two input fields to exist in input ev This event format is supported by the Logstash `syslog` input plugin out of the box, but several other plugins produce input that can be adapted fairly easy to produce these fields too. See [ALTERNATIVE INPUTS](ALTERNATIVE-INPUTS.md) for details. -Aggregation filter ------ - -A optional aggregation filter is available, that will combine fields from different log lines. The key on which log lines are aggregated is the postfix queue id. For example: - -![Alt text](aggregation_example_pic.jpg?raw=true) - -In this example, the `postfix_from` field from a `postfix/qmgr` log line is reused, and added to a log line from `postfix/smtp`. - -To use the aggregation filter, add `51-filter-postfix-aggregate.conf` to `/etc/logstash/conf.d` or `pipeline` dir for dockerized Logstash. - - Tests ----- -[![Build Status](https://travis-ci.org/whyscream/postfix-grok-patterns.svg?branch=master)](https://travis-ci.org/whyscream/postfix-grok-patterns) - In the `test/` directory, there is a test suite that tries to make sure that no previously supported log line will break because of changing common patterns and such. It also returns results a lot faster than doing `sudo service logstash restart` :-). -The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you also need `ruby 2.2` or higher, and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. +The test suite needs the patterns provided by Logstash, you can easily pull these from github by running `git submodule update --init`. To run the test suite, you need a recent version of `ruby` (`2.6` or newer should work), and the `jls-grok` and `minitest` gems. Then simply execute `ruby test/test.rb`. NOTE: The whole test process can now be executed inside a docker container, simply by running the `test_grok_patterns.sh` script. Adding new test cases can easily be done by creating new yaml files in the test directory. Each file specifies a grok pattern to validate, a sample log line, and a list of expected results. Also, the example Logstash config file adds some informative tags that aid in finding grok failures and unparsed lines. If you're not interested in those, you can remove all occurrences of `add_tag` and `tag_on_failure` from the config file. +Additional test scripts are available for local tests (using docker containers): +- `test_grok_patterns.sh`: runs the test suite for the grok patterns in `postfix.grok` +- `test_logstash_config.sh`: validates the logstash config in `50-filter-postfix.conf` +- `test_pipeline.sh`: validates that the logstash config can be used in a simple logstash pipeline, and ensures that this results in parsed messages + Contributing ------------ @@ -56,7 +47,7 @@ Other guidelines: License ------- -Everything in this repository is available under the New (3-clause) BSD license. +Everything in this repository is available under the New (3-clause) BSD license. See ![LICENSE](LICENSE) for details. Acknowledgement --------------- diff --git a/aggregation_example_pic.jpg b/aggregation_example_pic.jpg deleted file mode 100644 index c99db70..0000000 Binary files a/aggregation_example_pic.jpg and /dev/null differ diff --git a/postfix.grok b/postfix.grok index b74a35d..b0343d0 100644 --- a/postfix.grok +++ b/postfix.grok @@ -1,57 +1,58 @@ -# Version: 1.0.0 +# helper patterns +GREEDYDATA_NO_COLON [^:]* +GREEDYDATA_NO_SEMICOLON [^;]* +GREEDYDATA_NO_BRACKET [^<>]* +STATUS_WORD [\w-]* +IP_UNKNOWN unknown # common postfix patterns POSTFIX_QUEUEID ([0-9A-F]{6,}|[0-9a-zA-Z]{12,}|NOQUEUE) -POSTFIX_CLIENT_INFO %{HOSTNAME:postfix_client_hostname}?\[%{IP:postfix_client_ip}\](:%{INT:postfix_client_port})? -POSTFIX_RELAY_INFO %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} -POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) -POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|reject|reject_warning) + +POSTFIX_CLIENT %{HOSTNAME:postfix_client_hostname}?\[(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\](:%{INT:postfix_client_port})? +POSTFIX_RELAY %{HOSTNAME:postfix_relay_hostname}?\[(%{IP:postfix_relay_ip}|%{DATA:postfix_relay_service})\](:%{INT:postfix_relay_port})?|%{WORD:postfix_relay_service} +POSTFIX_SMTP_STAGE (CONNECT|HELO|EHLO|STARTTLS|AUTH|MAIL( FROM)?|RCPT( TO)?|(end of )?DATA|BDAT|RSET|UNKNOWN|END-OF-MESSAGE|VRFY|\.) +POSTFIX_ACTION (accept|defer|discard|filter|header-redirect|milter-reject|reject|reject_warning) POSTFIX_STATUS_CODE \d{3} POSTFIX_STATUS_CODE_ENHANCED \d\.\d+\.\d+ POSTFIX_DNSBL_MESSAGE Service unavailable; .* \[%{GREEDYDATA:postfix_status_data}\] %{GREEDYDATA:postfix_status_message}; -POSTFIX_PS_ACCESS_ACTION (DISCONNECT|BLACKLISTED|WHITELISTED|WHITELIST VETO|PASS NEW|PASS OLD) +POSTFIX_PS_ACCESS_ACTION (DISCONNECT|DENYLISTED|BLACKLISTED|ALLOWLISTED|WHITELISTED|ALLOWLIST VETO|WHITELIST VETO|PASS NEW|PASS OLD) POSTFIX_PS_VIOLATION (BARE NEWLINE|COMMAND (TIME|COUNT|LENGTH) LIMIT|COMMAND PIPELINING|DNSBL|HANGUP|NON-SMTP COMMAND|PREGREET) -POSTFIX_TIME_UNIT %{NUMBER}[smhd] POSTFIX_KEYVALUE_DATA [\w-]+=[^;]* POSTFIX_KEYVALUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} -POSTFIX_WARNING_LEVEL (warning|fatal|info) -POSTFIX_TLSCONN (Anonymous|Trusted|Untrusted|Verified) TLS connection established (to %{POSTFIX_RELAY_INFO}|from %{POSTFIX_CLIENT_INFO}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\) -POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY_INFO}: %{GREEDYDATA:postfix_tls_error} + +POSTFIX_TLSCONN %{DATA:postfix_tls_trustlevel} TLS connection established (to %{POSTFIX_RELAY}|from %{POSTFIX_CLIENT}): %{DATA:postfix_tls_version} with cipher %{DATA:postfix_tls_cipher} \(%{DATA:postfix_tls_cipher_size} bits\)( key-exchange %{DATA:postfix_tls_key_exchange} server-signature %{DATA:postfix_tls_server_signature} \((%{INT:postfix_tls_server_signature_size} bits|(?[PBK]-\d+))\)( server-digest %{DATA:postfix_tls_server_digest})?)? +POSTFIX_TLSVERIFICATION certificate verification failed for %{POSTFIX_RELAY}: %{GREEDYDATA:postfix_tls_error} POSTFIX_DELAYS %{NUMBER:postfix_delay_before_qmgr}/%{NUMBER:postfix_delay_in_qmgr}/%{NUMBER:postfix_delay_conn_setup}/%{NUMBER:postfix_delay_transmission} -POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1) -POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once) +POSTFIX_LOSTCONN (Connection timed out|No route to host|Connection refused|Network is unreachable|lost connection|timeout|SSL_accept error|-1|Address not available|Operation timed out|Address not available|Operation timed out|bare received) +POSTFIX_LOSTCONN_REASONS (receiving the initial server greeting|sending message body|sending end of data -- message may be sent more than once|sending %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}) POSTFIX_PROXY_MESSAGE (%{POSTFIX_STATUS_CODE:postfix_proxy_status_code} )?(%{POSTFIX_STATUS_CODE_ENHANCED:postfix_proxy_status_code_enhanced})?.* -POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} +POSTFIX_COMMAND_COUNTER_DATA (helo=(%{INT:postfix_cmd_helo_accepted}/)?%{INT:postfix_cmd_helo} )?(ehlo=(%{INT:postfix_cmd_ehlo_accepted}/)?%{INT:postfix_cmd_ehlo} )?(starttls=(%{INT:postfix_cmd_starttls_accepted}/)?%{INT:postfix_cmd_starttls} )?(auth=(%{INT:postfix_cmd_auth_accepted}/)?%{INT:postfix_cmd_auth} )?(mail=(%{INT:postfix_cmd_mail_accepted}/)?%{INT:postfix_cmd_mail} )?(rcpt=(%{INT:postfix_cmd_rcpt_accepted}/)?%{INT:postfix_cmd_rcpt} )?(bdat=(%{INT:postfix_cmd_bdat_accepted}/)?%{INT:postfix_cmd_bdat} )?(data=(%{INT:postfix_cmd_data_accepted}/)?%{INT:postfix_cmd_data} )?(rset=(%{INT:postfix_cmd_rset_accepted}/)?%{INT:postfix_cmd_rset} )?(noop=(%{INT:postfix_cmd_noop_accepted}/)?%{INT:postfix_cmd_noop} )?(quit=(%{INT:postfix_cmd_quit_accepted}/)?%{INT:postfix_cmd_quit} )?(unknown=(%{INT:postfix_cmd_unknown_accepted}/)?%{INT:postfix_cmd_unknown} )?commands=(%{INT:postfix_cmd_count_accepted}/)?%{INT:postfix_cmd_count} -# helper patterns -GREEDYDATA_NO_COLON [^:]* -GREEDYDATA_NO_SEMICOLON [^;]* -GREEDYDATA_NO_BRACKET [^<>]* -STATUS_WORD [\w-]* # warning patterns -POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} -POSTFIX_WARNING_WITHOUT_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?%{POSTFIX_WARNING_LEVEL:postfix_message_level}: (%{POSTFIX_CLIENT_INFO}: )?%{GREEDYDATA:postfix_message} +POSTFIX_WARNING_WITH_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?(?(warning|fatal|info)): (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT}: )?%{GREEDYDATA:postfix_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} +POSTFIX_WARNING_WITHOUT_KV (%{POSTFIX_QUEUEID:postfix_queueid}: )?(?(warning|fatal|info)): (%{POSTFIX_QUEUEID:postfix_queueid}: )?(%{POSTFIX_CLIENT}: )?%{GREEDYDATA:postfix_message} POSTFIX_WARNING %{POSTFIX_WARNING_WITH_KV}|%{POSTFIX_WARNING_WITHOUT_KV} # smtpd patterns -POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT_INFO} -POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT_INFO}( %{GREEDYDATA:postfix_command_counter_data})? -POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT_INFO}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})? -POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} -POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_improper_pipelining_data} +POSTFIX_SMTPD_CONNECT connect from %{POSTFIX_CLIENT} +POSTFIX_SMTPD_DISCONNECT disconnect from %{POSTFIX_CLIENT}( %{GREEDYDATA:postfix_command_counter_data})? +POSTFIX_SMTPD_LOSTCONN %{POSTFIX_LOSTCONN:postfix_smtpd_lostconn_data}( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage}( \(%{INT} bytes\))?)? from %{POSTFIX_CLIENT}(: %{GREEDYDATA:postfix_smtpd_lostconn_reason})? +POSTFIX_SMTPD_NOQUEUE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_ACTION:postfix_action}: %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT}:( %{POSTFIX_STATUS_CODE:postfix_status_code} %{POSTFIX_STATUS_CODE_ENHANCED:postfix_status_code_enhanced})?( <%{DATA:postfix_status_data}>:)? (%{POSTFIX_DNSBL_MESSAGE}|%{GREEDYDATA:postfix_status_message};) %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} +POSTFIX_SMTPD_PIPELINING improper command pipelining after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} from %{POSTFIX_CLIENT}: %{GREEDYDATA:postfix_improper_pipelining_data} POSTFIX_SMTPD_PROXY proxy-%{POSTFIX_ACTION:postfix_proxy_result}: (%{POSTFIX_SMTP_STAGE:postfix_proxy_smtp_stage}): %{POSTFIX_PROXY_MESSAGE:postfix_proxy_message}; %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} # cleanup patterns POSTFIX_CLEANUP_MILTER %{POSTFIX_QUEUEID:postfix_queueid}: milter-%{POSTFIX_ACTION:postfix_milter_result}: %{GREEDYDATA:postfix_milter_message}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}(: %{GREEDYDATA:postfix_milter_data})? POSTFIX_CLEANUP_PREPEND_TYPE (header|body) -POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT_INFO}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value} +POSTFIX_CLEANUP_PREPEND %{POSTFIX_QUEUEID:postfix_queueid}: prepend: %{POSTFIX_CLEANUP_PREPEND_TYPE:postfix_prepend_type} %{GREEDYDATA:postfix_prepend_trigger} from %{POSTFIX_CLIENT}; %{GREEDYDATA_NO_COLON:postfix_keyvalue_data}: %{GREEDYDATA:postfix_prepend_value} POSTFIX_CLEANUP_MESSAGEID %{POSTFIX_QUEUEID:postfix_queueid}: message-id=? # qmgr patterns -POSTFIX_QMGR_REMOVED %{POSTFIX_QUEUEID:postfix_queueid}: removed +POSTFIX_QMGR_MESSAGE (removed|skipped, still being delivered) +POSTFIX_QMGR_INFO %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_QMGR_MESSAGE:postfix_message} POSTFIX_QMGR_ACTIVE %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_KEYVALUE_DATA:postfix_keyvalue_data} \(queue active\) POSTFIX_QMGR_EXPIRED %{POSTFIX_QUEUEID:postfix_queueid}: from=<%{DATA:postfix_from}>, status=%{STATUS_WORD:postfix_status}, returned to sender @@ -71,57 +72,64 @@ POSTFIX_POSTSUPER_SUMMARY_ACTIONS (Deleted|Requeued|Placed on hold|Released from POSTFIX_POSTSUPER_SUMMARY %{POSTFIX_POSTSUPER_SUMMARY_ACTIONS:postfix_postsuper_summary_action}: %{NUMBER:postfix_postsuper_summary_count} messages? # postscreen patterns -POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT_INFO} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port} -POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT_INFO} +POSTFIX_PS_CONNECT CONNECT from %{POSTFIX_CLIENT} to \[%{IP:postfix_server_ip}\]:%{INT:postfix_server_port} +POSTFIX_PS_ACCESS %{POSTFIX_PS_ACCESS_ACTION:postfix_postscreen_access} %{POSTFIX_CLIENT} POSTFIX_PS_NOQUEUE %{POSTFIX_SMTPD_NOQUEUE} -POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT_INFO}: %{GREEDYDATA:postfix_postscreen_toobusy_data} -POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT_INFO} +POSTFIX_PS_TOOBUSY NOQUEUE: reject: CONNECT from %{POSTFIX_CLIENT}: %{GREEDYDATA:postfix_postscreen_toobusy_data} +POSTFIX_PS_DNSBL %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation} rank %{INT:postfix_postscreen_dnsbl_rank} for %{POSTFIX_CLIENT} POSTFIX_PS_CACHE cache %{DATA} full cleanup: retained=%{NUMBER:postfix_postscreen_cache_retained} dropped=%{NUMBER:postfix_postscreen_cache_dropped} entries -POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT_INFO}(( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?(: %{GREEDYDATA:postfix_postscreen_data})?| in tests (after|before) SMTP handshake) +POSTFIX_PS_VIOLATIONS %{POSTFIX_PS_VIOLATION:postfix_postscreen_violation}( %{INT})?( after %{NUMBER:postfix_postscreen_violation_time})? from %{POSTFIX_CLIENT}(( after %{POSTFIX_SMTP_STAGE:postfix_smtp_stage})?(: %{GREEDYDATA:postfix_postscreen_data})?| in tests (after|before) SMTP handshake) # dnsblog patterns POSTFIX_DNSBLOG_LISTING addr %{IP:postfix_client_ip} listed by domain %{HOSTNAME:postfix_dnsbl_domain} as %{IP:postfix_dnsbl_result} # tlsproxy patterns -POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT_INFO} +POSTFIX_TLSPROXY_CONN (DIS)?CONNECT( from)? %{POSTFIX_CLIENT} # anvil patterns -POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/%{POSTFIX_TIME_UNIT:postfix_anvil_conn_period} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} +POSTFIX_ANVIL_CONN_RATE statistics: max connection rate %{NUMBER:postfix_anvil_conn_rate}/(?\d+[smhd]) for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} POSTFIX_ANVIL_CONN_CACHE statistics: max cache size %{NUMBER:postfix_anvil_cache_size} at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} -POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:%{IP:postfix_client_ip}\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} +POSTFIX_ANVIL_CONN_COUNT statistics: max connection count %{NUMBER:postfix_anvil_conn_count} for \(%{DATA:postfix_service}:(%{IP_UNKNOWN:postfix_client_ip_unknown}|%{IP:postfix_client_ip})\) at %{SYSLOGTIMESTAMP:postfix_anvil_timestamp} # smtp patterns POSTFIX_SMTP_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_smtp_response}\))? -POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} -POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY_INFO}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} -POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY_INFO}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? -POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY_INFO} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? -POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\) -POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY_INFO} said: %{GREEDYDATA:postfix_smtp_response} -POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY_INFO} offers SMTPUTF8 support, but not 8BITMIME -POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY_INFO} +POSTFIX_SMTP_CONNERR connect to %{POSTFIX_RELAY}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} +POSTFIX_SMTP_SSLCONNERR SSL_connect error to %{POSTFIX_RELAY}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} +POSTFIX_SMTP_LOSTCONN %{POSTFIX_QUEUEID:postfix_queueid}: %{POSTFIX_LOSTCONN:postfix_smtp_lostconn_data} with %{POSTFIX_RELAY}( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? +POSTFIX_SMTP_TIMEOUT %{POSTFIX_QUEUEID:postfix_queueid}: conversation with %{POSTFIX_RELAY} timed out( while %{POSTFIX_LOSTCONN_REASONS:postfix_smtp_lostconn_reason})? +POSTFIX_SMTP_RELAYERR %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY} said: %{GREEDYDATA:postfix_smtp_response} \(in reply to %{POSTFIX_SMTP_STAGE:postfix_smtp_stage} command\) +POSTFIX_SMTP_RELAYREFUSED %{POSTFIX_QUEUEID:postfix_queueid}: host %{POSTFIX_RELAY} refused to talk to me: %{GREEDYDATA:postfix_smtp_response} +POSTFIX_SMTP_SSLAUTHERR %{POSTFIX_QUEUEID:postfix_queueid}: SASL authentication failed; server %{POSTFIX_RELAY} said: %{GREEDYDATA:postfix_smtp_response} +POSTFIX_SMTP_UTF8 host %{POSTFIX_RELAY} offers SMTPUTF8 support, but not 8BITMIME +POSTFIX_SMTP_PIX %{POSTFIX_QUEUEID:postfix_queueid}: enabling PIX workarounds: %{DATA:postfix_pix_workaround} for %{POSTFIX_RELAY} # master patterns POSTFIX_MASTER_START (daemon started|reload) -- version %{DATA:postfix_version}, configuration %{PATH:postfix_config_path} POSTFIX_MASTER_EXIT terminating on signal %{INT:postfix_termination_signal} # bounce patterns -POSTFIX_BOUNCE_NOTIFICATION %{POSTFIX_QUEUEID:postfix_queueid}: sender (non-delivery|delivery status|delay) notification: %{POSTFIX_QUEUEID:postfix_bounce_queueid} +POSTFIX_BOUNCE_NOTIFICATION %{POSTFIX_QUEUEID:postfix_queueid}: (?(sender|postmaster)) (?(non-delivery|delivery status|delay)) notification: %{POSTFIX_QUEUEID:postfix_bounce_queueid} # scache patterns POSTFIX_SCACHE_LOOKUPS statistics: (address|domain) lookup hits=%{INT:postfix_scache_hits} miss=%{INT:postfix_scache_miss} success=%{INT:postfix_scache_success}% POSTFIX_SCACHE_SIMULTANEOUS statistics: max simultaneous domains=%{INT:postfix_scache_domains} addresses=%{INT:postfix_scache_addresses} connection=%{INT:postfix_scache_connection} POSTFIX_SCACHE_TIMESTAMP statistics: start interval %{SYSLOGTIMESTAMP:postfix_scache_timestamp} +# verify patterns +POSTFIX_VERIFY_CACHE cache %{DATA} (?(full|partial)) cleanup: retained=%{INT:postfix_verify_cache_retained} dropped=%{INT:postfix_verify_cache_dropped} entries + +# local patterns +POSTFIX_LOCAL_DELIVERY %{POSTFIX_KEYVALUE} status=%{STATUS_WORD:postfix_status}( \(%{GREEDYDATA:postfix_local_response}\))? + # aggregate all patterns POSTFIX_SMTPD %{POSTFIX_SMTPD_CONNECT}|%{POSTFIX_SMTPD_DISCONNECT}|%{POSTFIX_SMTPD_LOSTCONN}|%{POSTFIX_SMTPD_NOQUEUE}|%{POSTFIX_SMTPD_PIPELINING}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTPD_PROXY}|%{POSTFIX_KEYVALUE} POSTFIX_CLEANUP %{POSTFIX_CLEANUP_MESSAGEID}|%{POSTFIX_CLEANUP_MILTER}|%{POSTFIX_CLEANUP_PREPEND}|%{POSTFIX_WARNING}|%{POSTFIX_KEYVALUE} -POSTFIX_QMGR %{POSTFIX_QMGR_REMOVED}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING} +POSTFIX_QMGR %{POSTFIX_QMGR_INFO}|%{POSTFIX_QMGR_ACTIVE}|%{POSTFIX_QMGR_EXPIRED}|%{POSTFIX_WARNING} POSTFIX_PIPE %{POSTFIX_PIPE_ANY} POSTFIX_POSTSCREEN %{POSTFIX_PS_CONNECT}|%{POSTFIX_PS_ACCESS}|%{POSTFIX_PS_NOQUEUE}|%{POSTFIX_PS_TOOBUSY}|%{POSTFIX_PS_CACHE}|%{POSTFIX_PS_DNSBL}|%{POSTFIX_PS_VIOLATIONS}|%{POSTFIX_WARNING} POSTFIX_DNSBLOG %{POSTFIX_DNSBLOG_LISTING}|%{POSTFIX_WARNING} POSTFIX_ANVIL %{POSTFIX_ANVIL_CONN_RATE}|%{POSTFIX_ANVIL_CONN_CACHE}|%{POSTFIX_ANVIL_CONN_COUNT} -POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX} +POSTFIX_SMTP %{POSTFIX_SMTP_DELIVERY}|%{POSTFIX_SMTP_CONNERR}|%{POSTFIX_SMTP_SSLCONNERR}|%{POSTFIX_SMTP_SSLAUTHERR}|%{POSTFIX_SMTP_LOSTCONN}|%{POSTFIX_SMTP_TIMEOUT}|%{POSTFIX_SMTP_RELAYERR}|%{POSTFIX_SMTP_RELAYREFUSED}|%{POSTFIX_TLSCONN}|%{POSTFIX_WARNING}|%{POSTFIX_SMTP_UTF8}|%{POSTFIX_TLSVERIFICATION}|%{POSTFIX_SMTP_PIX} POSTFIX_DISCARD %{POSTFIX_DISCARD_ANY}|%{POSTFIX_WARNING} POSTFIX_LMTP %{POSTFIX_SMTP} POSTFIX_PICKUP %{POSTFIX_KEYVALUE} @@ -133,9 +141,10 @@ POSTFIX_POSTDROP %{POSTFIX_WARNING} POSTFIX_SCACHE %{POSTFIX_SCACHE_LOOKUPS}|%{POSTFIX_SCACHE_SIMULTANEOUS}|%{POSTFIX_SCACHE_TIMESTAMP} POSTFIX_TRIVIAL_REWRITE %{POSTFIX_WARNING} POSTFIX_TLSMGR %{POSTFIX_WARNING} -POSTFIX_LOCAL %{POSTFIX_KEYVALUE}|%{POSTFIX_WARNING} +POSTFIX_LOCAL %{POSTFIX_LOCAL_DELIVERY}|%{POSTFIX_WARNING} POSTFIX_VIRTUAL %{POSTFIX_SMTP_DELIVERY} POSTFIX_ERROR %{POSTFIX_ERROR_ANY} POSTFIX_POSTSUPER %{POSTFIX_POSTSUPER_ACTION}|%{POSTFIX_POSTSUPER_SUMMARY} POSTFIX_POSTMAP %{POSTFIX_WARNING} POSTFIX_SCRIPT %{POSTFIX_WARNING} +POSTFIX_VERIFY %{POSTFIX_VERIFY_CACHE} diff --git a/test/anvil_0005.yaml b/test/anvil_0005.yaml index bcc5748..10abe32 100644 --- a/test/anvil_0005.yaml +++ b/test/anvil_0005.yaml @@ -1,8 +1,8 @@ pattern: ^%{POSTFIX_ANVIL}$ -data: "statistics: max connection rate 1/60s for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59" +data: "statistics: max connection rate 1/5m for (smtpd:2604:8d00:0:1::3) at Oct 26 17:46:59" results: postfix_anvil_conn_rate: 1 - postfix_anvil_conn_period: 60s + postfix_anvil_conn_period: 5m postfix_service: smtpd postfix_client_ip: 2604:8d00:0:1::3 postfix_anvil_timestamp: Oct 26 17:46:59 diff --git a/test/anvil_0007.yaml b/test/anvil_0007.yaml index 27b01d3..96113ff 100644 --- a/test/anvil_0007.yaml +++ b/test/anvil_0007.yaml @@ -1,8 +1,8 @@ pattern: ^%{POSTFIX_ANVIL}$ -data: "statistics: max connection rate 1/60s for (127.0.0.1:2525:127.0.0.1) at Oct 26 18:13:50" +data: "statistics: max connection rate 1/2h for (127.0.0.1:2525:127.0.0.1) at Oct 26 18:13:50" results: postfix_anvil_conn_rate: 1 - postfix_anvil_conn_period: 60s + postfix_anvil_conn_period: 2h postfix_service: 127.0.0.1:2525 postfix_client_ip: 127.0.0.1 postfix_anvil_timestamp: Oct 26 18:13:50 diff --git a/test/anvil_0008.yaml b/test/anvil_0008.yaml new file mode 100644 index 0000000..e84caf4 --- /dev/null +++ b/test/anvil_0008.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_ANVIL}$ +data: "statistics: max connection count 1 for (smtp:unknown) at Sep 7 07:14:19" +results: + postfix_anvil_conn_count: 1 + postfix_service: smtp + postfix_client_ip_unknown: unknown + postfix_anvil_timestamp: Sep 7 07:14:19 diff --git a/test/anvil_0009.yaml b/test/anvil_0009.yaml new file mode 100644 index 0000000..ae5db7a --- /dev/null +++ b/test/anvil_0009.yaml @@ -0,0 +1,8 @@ +pattern: ^%{POSTFIX_ANVIL}$ +data: "statistics: max connection rate 1/7d for (smtp:unknown) at Sep 7 07:14:19" +results: + postfix_anvil_conn_rate: 1 + postfix_anvil_conn_period: 7d + postfix_service: smtp + postfix_client_ip_unknown: unknown + postfix_anvil_timestamp: Sep 7 07:14:19 diff --git a/test/bounce_0001.yaml b/test/bounce_0001.yaml index 4480694..648cc04 100644 --- a/test/bounce_0001.yaml +++ b/test/bounce_0001.yaml @@ -3,3 +3,5 @@ data: "17CCA8044: sender non-delivery notification: ADD2F8052" results: postfix_queueid: 17CCA8044 postfix_bounce_queueid: ADD2F8052 + postfix_bounce_recipient: sender + postfix_bounce_reason: non-delivery diff --git a/test/bounce_0002.yaml b/test/bounce_0002.yaml index 025bde9..7120e93 100644 --- a/test/bounce_0002.yaml +++ b/test/bounce_0002.yaml @@ -3,3 +3,5 @@ data: "65AFB8B5: sender delivery status notification: 1DF35917" results: postfix_queueid: 65AFB8B5 postfix_bounce_queueid: 1DF35917 + postfix_bounce_recipient: sender + postfix_bounce_reason: delivery status diff --git a/test/bounce_0003.yaml b/test/bounce_0003.yaml index b54f640..7c3e648 100644 --- a/test/bounce_0003.yaml +++ b/test/bounce_0003.yaml @@ -3,3 +3,5 @@ data: "264FE1A18: sender delay notification: 0A87A1A08" results: postfix_queueid: 264FE1A18 postfix_bounce_queueid: 0A87A1A08 + postfix_bounce_recipient: sender + postfix_bounce_reason: delay diff --git a/test/bounce_0004.yaml b/test/bounce_0004.yaml new file mode 100644 index 0000000..62b61ee --- /dev/null +++ b/test/bounce_0004.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_BOUNCE}$ +data: "482E4404AC8: postmaster non-delivery notification: 47335405A44" +results: + postfix_queueid: 482E4404AC8 + postfix_bounce_queueid: 47335405A44 + postfix_bounce_recipient: postmaster + postfix_bounce_reason: non-delivery diff --git a/test/cleanup_0012.yaml b/test/cleanup_0012.yaml new file mode 100644 index 0000000..c6b9037 --- /dev/null +++ b/test/cleanup_0012.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_CLEANUP}$ +data: "warning: 9BECF3F19D: sender_canonical_maps map lookup problem for sender@example.com -- message not accepted, try again later" +results: + postfix_queueid: 9BECF3F19D + postfix_message_level: warning + postfix_message: "sender_canonical_maps map lookup problem for sender@example.com -- message not accepted, try again later" diff --git a/test/local_0001.yaml b/test/local_0001.yaml index c2be9b0..9d9d091 100644 --- a/test/local_0001.yaml +++ b/test/local_0001.yaml @@ -2,4 +2,6 @@ pattern: ^%{POSTFIX_LOCAL}$ data: "2A22C263F6: to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")" results: postfix_queueid: 2A22C263F6 - postfix_keyvalue_data: "to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0, status=sent (delivered to command: procmail -a \"$EXTENSION\")" + postfix_keyvalue_data: "to=user@hostname.example.com, orig_to=root@localhost, relay=local, delay=0.07, delays=0.04/0/0/0.03, dsn=2.0.0," + postfix_status: sent + postfix_local_response: "delivered to command: procmail -a \"$EXTENSION\"" diff --git a/test/local_0002.yaml b/test/local_0002.yaml index 9a477c7..b2924a0 100644 --- a/test/local_0002.yaml +++ b/test/local_0002.yaml @@ -2,4 +2,6 @@ pattern: ^%{POSTFIX_LOCAL}$ data: "892A0205B6: to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")" results: postfix_queueid: 892A0205B6 - postfix_keyvalue_data: "to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1, status=bounced (unknown user: \"ghdsgfhdslfh\")" + postfix_keyvalue_data: "to=ghdsgfhdslfh@localhost, relay=local, delay=0.05, delays=0.02/0/0/0.02, dsn=5.1.1," + postfix_status: bounced + postfix_local_response: "unknown user: \"ghdsgfhdslfh\"" diff --git a/test/postscreen_0030.yaml b/test/postscreen_0030.yaml new file mode 100644 index 0000000..5756d9b --- /dev/null +++ b/test/postscreen_0030.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_POSTSCREEN}$ +data: "ALLOWLIST VETO [61.238.241.86]:53024" +results: + postfix_postscreen_access: ALLOWLIST VETO + postfix_client_ip: 61.238.241.86 + postfix_client_port: 53024 diff --git a/test/qmgr_0001.yaml b/test/qmgr_0001.yaml index 64f9dc0..34271cd 100644 --- a/test/qmgr_0001.yaml +++ b/test/qmgr_0001.yaml @@ -2,3 +2,4 @@ pattern: ^%{POSTFIX_QMGR}$ data: "0F5383D: removed" results: postfix_queueid: 0F5383D + postfix_message: removed diff --git a/test/qmgr_0006.yaml b/test/qmgr_0006.yaml new file mode 100644 index 0000000..551f97b --- /dev/null +++ b/test/qmgr_0006.yaml @@ -0,0 +1,5 @@ +pattern: ^%{POSTFIX_QMGR}$ +data: "ED06590130AA1C9D6D862: skipped, still being delivered" +results: + postfix_queueid: ED06590130AA1C9D6D862 + postfix_message: skipped, still being delivered diff --git a/test/relay_info_0001.yaml b/test/relay_0001.yaml similarity index 74% rename from test/relay_info_0001.yaml rename to test/relay_0001.yaml index 926f5eb..7133938 100644 --- a/test/relay_info_0001.yaml +++ b/test/relay_0001.yaml @@ -1,5 +1,5 @@ -# this data is derived after the keyvalue data is expanded in logstash to a 'postfix_relay_info' field. -pattern: ^%{POSTFIX_RELAY_INFO}$ +# this data is derived after the keyvalue data is expanded in logstash to a 'postfix_relay' field. +pattern: ^%{POSTFIX_RELAY}$ data: "1.example.com.si[private/dovecot-lmtp]" results: postfix_relay_hostname: 1.example.com.si diff --git a/test/smtp_0003.yaml b/test/smtp_0003.yaml index 81dd6f1..c5bb6ca 100644 --- a/test/smtp_0003.yaml +++ b/test/smtp_0003.yaml @@ -1,6 +1,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "Untrusted TLS connection established to mx4.hotmail.com[65.55.92.136]:25: TLSv1.2 with cipher ECDHE-RSA-AES256-SHA384 (256/256 bits)" results: + postfix_tls_trustlevel: Untrusted postfix_relay_hostname: mx4.hotmail.com postfix_relay_ip: 65.55.92.136 postfix_relay_port: 25 diff --git a/test/smtp_0004.yaml b/test/smtp_0004.yaml index c33f292..0fc742d 100644 --- a/test/smtp_0004.yaml +++ b/test/smtp_0004.yaml @@ -1,6 +1,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "Untrusted TLS connection established to 127.0.0.1[127.0.0.1]:2525: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)" results: + postfix_tls_trustlevel: Untrusted postfix_relay_hostname: 127.0.0.1 postfix_relay_ip: 127.0.0.1 postfix_relay_port: 2525 diff --git a/test/smtp_0006.yaml b/test/smtp_0006.yaml index 5912607..84b580f 100644 --- a/test/smtp_0006.yaml +++ b/test/smtp_0006.yaml @@ -2,5 +2,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "B99FE3D: lost connection with mx3.hotmail.com[65.55.37.72] while receiving the initial server greeting" results: postfix_queueid: B99FE3D + postfix_smtp_lostconn_data: lost connection + postfix_smtp_lostconn_reason: receiving the initial server greeting postfix_relay_hostname: mx3.hotmail.com postfix_relay_ip: 65.55.37.72 diff --git a/test/smtp_0015.yaml b/test/smtp_0015.yaml index 3f7028e..12c2e50 100644 --- a/test/smtp_0015.yaml +++ b/test/smtp_0015.yaml @@ -1,6 +1,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "Trusted TLS connection established to gmail-smtp-in.l.google.com[74.125.136.26]:25: TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)" results: + postfix_tls_trustlevel: Trusted postfix_relay_hostname: gmail-smtp-in.l.google.com postfix_relay_ip: 74.125.136.26 postfix_relay_port: 25 diff --git a/test/smtp_0016.yaml b/test/smtp_0016.yaml index f3c6344..35b3858 100644 --- a/test/smtp_0016.yaml +++ b/test/smtp_0016.yaml @@ -1,6 +1,7 @@ pattern: ^%{POSTFIX_SMTP}$ data: "Verified TLS connection established to mail.sys4.de[2001:1578:400:111::7]:25: TLSv1 with cipher ECDHE-RSA-AES256-SHA (256/256 bits)" results: + postfix_tls_trustlevel: Verified postfix_relay_hostname: mail.sys4.de postfix_relay_ip: 2001:1578:400:111::7 postfix_relay_port: 25 diff --git a/test/smtp_0033.yaml b/test/smtp_0033.yaml new file mode 100644 index 0000000..9ee72d0 --- /dev/null +++ b/test/smtp_0033.yaml @@ -0,0 +1,8 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "745137AA: lost connection with hotmail-com.olc.protection.outlook.com[104.47.55.161] while sending RCPT TO" +results: + postfix_queueid: 745137AA + postfix_smtp_lostconn_data: lost connection + postfix_smtp_lostconn_reason: sending RCPT TO + postfix_relay_hostname: hotmail-com.olc.protection.outlook.com + postfix_relay_ip: 104.47.55.161 diff --git a/test/smtp_0034.yaml b/test/smtp_0034.yaml new file mode 100644 index 0000000..7fb4046 --- /dev/null +++ b/test/smtp_0034.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "40197AA: host mg2.egov.bg[213.91.191.86] refused to talk to me: 450 4.3.2 try again later" +results: + postfix_queueid: 40197AA + postfix_relay_hostname: mg2.egov.bg + postfix_relay_ip: 213.91.191.86 + postfix_smtp_response: 450 4.3.2 try again later diff --git a/test/smtp_0035.yaml b/test/smtp_0035.yaml new file mode 100644 index 0000000..af0974b --- /dev/null +++ b/test/smtp_0035.yaml @@ -0,0 +1,14 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "Untrusted TLS connection established to gmail-smtp-in.l.google.com[74.125.128.27]:25: TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature ECDSA (P-256) server-digest SHA256" +results: + postfix_tls_trustlevel: Untrusted + postfix_relay_hostname: gmail-smtp-in.l.google.com + postfix_relay_ip: 74.125.128.27 + postfix_relay_port: 25 + postfix_tls_version: TLSv1.3 + postfix_tls_cipher: TLS_AES_256_GCM_SHA384 + postfix_tls_cipher_size: 256/256 + postfix_tls_key_exchange: X25519 + postfix_tls_server_signature: ECDSA + postfix_tls_server_signature_curve: P-256 + postfix_tls_server_digest: SHA256 diff --git a/test/smtp_0036.yaml b/test/smtp_0036.yaml new file mode 100644 index 0000000..b802517 --- /dev/null +++ b/test/smtp_0036.yaml @@ -0,0 +1,13 @@ +pattern: ^%{POSTFIX_SMTP}$ +data: "Trusted TLS connection established to mail.example.com[127.0.0.1]:25: TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits)" +results: + postfix_tls_trustlevel: Trusted + postfix_relay_hostname: mail.example.com + postfix_relay_ip: 127.0.0.1 + postfix_relay_port: 25 + postfix_tls_version: TLSv1.3 + postfix_tls_cipher: TLS_AES_256_GCM_SHA384 + postfix_tls_cipher_size: 256/256 + postfix_tls_key_exchange: X25519 + postfix_tls_server_signature: RSA-PSS + postfix_tls_server_signature_size: 4096 diff --git a/test/smtpd_0010.yaml b/test/smtpd_0010.yaml index 70cadd7..9762ddf 100644 --- a/test/smtpd_0010.yaml +++ b/test/smtpd_0010.yaml @@ -1,6 +1,7 @@ pattern: ^%{POSTFIX_SMTPD}$ data: "Anonymous TLS connection established from julie.example.com[10.163.89.202]: TLSv1.2 with cipher AECDH-AES256-SHA (256/256 bits)" results: + postfix_tls_trustlevel: Anonymous postfix_client_hostname: julie.example.com postfix_client_ip: 10.163.89.202 postfix_tls_version: TLSv1.2 diff --git a/test/smtpd_0028.yaml b/test/smtpd_0028.yaml index 89911a3..fc8a348 100644 --- a/test/smtpd_0028.yaml +++ b/test/smtpd_0028.yaml @@ -4,4 +4,3 @@ results: postfix_client_hostname: unknown postfix_client_ip: 72.10.165.66 postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 data=0/1 quit=1 commands=5/7 - diff --git a/test/smtpd_0032.yaml b/test/smtpd_0032.yaml new file mode 100644 index 0000000..9f35f16 --- /dev/null +++ b/test/smtpd_0032.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from unknown[unknown] commands=0/0" +results: + postfix_client_hostname: unknown + postfix_client_ip_unknown: unknown + postfix_command_counter_data: commands=0/0 + diff --git a/test/smtpd_0033.yaml b/test/smtpd_0033.yaml new file mode 100644 index 0000000..a69263e --- /dev/null +++ b/test/smtpd_0033.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "lost connection after BDAT from example.net[158.247.23.50]" +results: + postfix_smtpd_lostconn_data: lost connection + postfix_smtp_stage: BDAT + postfix_client_hostname: example.net + postfix_client_ip: 158.247.23.50 diff --git a/test/smtpd_0034.yaml b/test/smtpd_0034.yaml new file mode 100644 index 0000000..c55e2bc --- /dev/null +++ b/test/smtpd_0034.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from example.net[158.247.23.50] ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6" +results: + postfix_client_hostname: example.net + postfix_client_ip: 158.247.23.50 + postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 bdat=0/1 commands=4/6 diff --git a/test/smtpd_0035.yaml b/test/smtpd_0035.yaml new file mode 100644 index 0000000..5a40658 --- /dev/null +++ b/test/smtpd_0035.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "disconnect from unknown[93.188.162.137] ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8" +results: + postfix_client_hostname: unknown + postfix_client_ip: 93.188.162.137 + postfix_command_counter_data: ehlo=2 starttls=1 mail=1 rcpt=0/1 rset=1 noop=1 quit=1 commands=7/8 diff --git a/test/smtpd_0036.yaml b/test/smtpd_0036.yaml new file mode 100644 index 0000000..9426412 --- /dev/null +++ b/test/smtpd_0036.yaml @@ -0,0 +1,7 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "bare received after CONNECT from example.net[1.2.3.4]" +results: + postfix_smtpd_lostconn_data: bare received + postfix_smtp_stage: CONNECT + postfix_client_hostname: example.net + postfix_client_ip: 1.2.3.4 diff --git a/test/smtpd_0037.yaml b/test/smtpd_0037.yaml new file mode 100644 index 0000000..b88105d --- /dev/null +++ b/test/smtpd_0037.yaml @@ -0,0 +1,13 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "Anonymous TLS connection established from julie.example.com[10.163.89.202]: TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (4096 bits) server-digest SHA256" +results: + postfix_tls_trustlevel: Anonymous + postfix_client_hostname: julie.example.com + postfix_client_ip: 10.163.89.202 + postfix_tls_version: TLSv1.3 + postfix_tls_cipher: TLS_AES_256_GCM_SHA384 + postfix_tls_cipher_size: 256/256 + postfix_tls_key_exchange: X25519 + postfix_tls_server_signature: RSA-PSS + postfix_tls_server_signature_size: 4096 + postfix_tls_server_digest: SHA256 diff --git a/test/smtpd_0038.yaml b/test/smtpd_0038.yaml new file mode 100644 index 0000000..4a91cf9 --- /dev/null +++ b/test/smtpd_0038.yaml @@ -0,0 +1,12 @@ +pattern: ^%{POSTFIX_SMTPD}$ +data: "NOQUEUE: milter-reject: RCPT from mailrelay.example.com[8.8.8.8]: 451 4.7.1 Greylisting in action, please come back later; from= to= proto=ESMTP helo=" +results: + postfix_queueid: NOQUEUE + postfix_status_code_enhanced: 4.7.1 + postfix_keyvalue_data: from= to= proto=ESMTP helo= + postfix_action: milter-reject + postfix_smtp_stage: RCPT + postfix_status_code: 451 + postfix_client_ip: 8.8.8.8 + postfix_status_message: "Greylisting in action, please come back later" + postfix_client_hostname: mailrelay.example.com diff --git a/test/test.rb b/test/test.rb index c807aca..0cf9bd9 100644 --- a/test/test.rb +++ b/test/test.rb @@ -12,7 +12,7 @@ # It creates test cases from all yaml files in the current # directory. -class TestGrokPatterns < MiniTest::Test +class TestGrokPatterns < Minitest::Test @@test_dir = File.dirname(__FILE__) @@upstream_pattern_dir = @@test_dir + '/logstash-patterns-core/patterns/ecs-v1/' diff --git a/test/verify_0001.yaml b/test/verify_0001.yaml new file mode 100644 index 0000000..0804e17 --- /dev/null +++ b/test/verify_0001.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_VERIFY}$ +data: "cache lmdb:/var/lib/postfix/verify_cache full cleanup: retained=724 dropped=6 entries" +results: + postfix_verify_cleanup_type: full + postfix_verify_cache_retained: 724 + postfix_verify_cache_dropped: 6 diff --git a/test/verify_0002.yaml b/test/verify_0002.yaml new file mode 100644 index 0000000..cf705d1 --- /dev/null +++ b/test/verify_0002.yaml @@ -0,0 +1,6 @@ +pattern: ^%{POSTFIX_VERIFY}$ +data: "cache ??????????????????????????????????? partial cleanup: retained=1 dropped=0 entries" +results: + postfix_verify_cleanup_type: partial + postfix_verify_cache_retained: 1 + postfix_verify_cache_dropped: 0 diff --git a/test_config_syntax.sh b/test_config_syntax.sh new file mode 100755 index 0000000..228e42d --- /dev/null +++ b/test_config_syntax.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# +# This script is used to test the config syntax of the 50-filter-postfix.conf file. +# +# The configuration file is validated using the logstash --config.test_and_exit command in a docker container. +# + +set -eu + +LOGSTASH_VERSION=8.14.1 + +docker run \ + --rm \ + --volume "$(pwd)"/postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume "$(pwd)"/50-filter-postfix.conf:/usr/share/logstash/pipeline/50-filter-postfix.conf \ + logstash:"$LOGSTASH_VERSION" \ + logstash --config.test_and_exit -f /usr/share/logstash/pipeline/50-filter-postfix.conf diff --git a/test_grok_patterns.sh b/test_grok_patterns.sh new file mode 100755 index 0000000..c3d111d --- /dev/null +++ b/test_grok_patterns.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +# +# This script is used to test the grok patterns in the postfix.grok file. +# +# The patterns are tested by running the test suite (in test/test.rb and test/*.yaml) +# against the patterns in the postfix.grok file in a docker container. +# +set -eu + +DOCKERIMAGE="postfix-grok-patterns-runtests" +VOLUMEPATH="/runtests" + +git submodule update --init + +docker build --tag ${DOCKERIMAGE} - < /dev/null + rm -f "$INPUT" "$OUTPUT" "$PIPELINE" +} +trap perform_cleanup INT TERM + +echo Preparing input data +echo "postfix/smtp[123]: 7EE668039: to=, relay=127.0.0.1[127.0.0.1]:2525, delay=3.6, delays=0.2/0.02/0.04/3.3, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as 153053D)" > "$INPUT" + +echo Preparing pipeline config +cat > "$PIPELINE" << EOF +input { + file { + path => "/tmp/logstash.in" + start_position => beginning + } +} +filter { + dissect { + mapping => { + "message" => "%{program}[%{pid}]: %{message}" + } + } +} +EOF + +cat 50-filter-postfix.conf >> "$PIPELINE" + +cat >> "$PIPELINE" << EOF +output { + file { + path => "/tmp/logstash.out" + } +} +EOF + +echo Starting logstash docker container +CONTAINER_ID=$(docker run --rm --detach \ + --volume ./"${INPUT}":/tmp/logstash.in \ + --volume ./"${OUTPUT}":/tmp/logstash.out \ + --volume ./postfix.grok:/etc/logstash/patterns.d/postfix.grok \ + --volume ./"${PIPELINE}":/usr/share/logstash/pipeline/pipeline.conf \ + logstash:"$LOGSTASH_VERSION" \ + logstash -f /usr/share/logstash/pipeline/pipeline.conf) + +printf "Waiting for output from logstash " +until test -s "$OUTPUT"; do + printf "." + sleep 2 +done +echo + +if test "$(jq .tags[0] "$OUTPUT")" = '"_grok_postfix_success"'; then + echo Grok processing successful! + jq . "$OUTPUT" +else + echo "Grok processing failed :<" + jq . "$OUTPUT" + exit 1 +fi + +perform_cleanup + +echo Done