diff --git a/Dockerfile b/Dockerfile
index 4ecc200..df6549e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,29 +4,30 @@ COPY . /src
WORKDIR /src
-ARG BUILD_DEPS="curl gcc g++ libpcap-dev"
-ARG OUI_SRC="http://standards-oui.ieee.org/oui.txt"
+ARG OUI_SRC="http://standards-oui.ieee.org/oui/oui.txt"
-RUN apk add --no-cache ${BUILD_DEPS} && python -m venv "/opt/venv"
+ENV VIRTUAL_ENV="/opt/venv"
-RUN curl --location --silent --output "/src/dshell/data/oui.txt" "${OUI_SRC}"
+RUN apk add cargo curl g++ gcc rust libpcap-dev libffi-dev \
+ && python3 -m venv "${VIRTUAL_ENV}" \
+ && curl --location --silent --output "/src/dshell/data/oui.txt" "${OUI_SRC}"
-ENV PATH="/opt/venv/bin:${PATH}"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
-RUN pip install --upgrade pip wheel && pip install --use-feature=2020-resolver .
+RUN pip install --upgrade pip wheel && pip install .
FROM python:3-alpine
-ARG RUN_DEPS="bash libstdc++ libpcap"
+ENV VIRTUAL_ENV="/opt/venv"
-COPY --from=builder /opt/venv /opt/venv
+COPY --from=builder "${VIRTUAL_ENV}/" "${VIRTUAL_ENV}/"
-RUN apk add --no-cache ${RUN_DEPS}
+RUN apk add --no-cache bash libstdc++ libpcap
VOLUME ["/data"]
WORKDIR "/data"
-ENV PATH="/opt/venv/bin:${PATH}"
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
ENTRYPOINT ["dshell"]
diff --git a/Dshell_Developer_Guide.pdf b/Dshell_Developer_Guide.pdf
new file mode 100644
index 0000000..dd5cb4e
Binary files /dev/null and b/Dshell_Developer_Guide.pdf differ
diff --git a/Dshell_User_Guide.pdf b/Dshell_User_Guide.pdf
new file mode 100644
index 0000000..21badf1
Binary files /dev/null and b/Dshell_User_Guide.pdf differ
diff --git a/README b/README
index 10a2f6a..29ea177 100644
--- a/README
+++ b/README
@@ -5,17 +5,29 @@ Key features:
* Deep packet analysis using specialized plugins
* Robust stream reassembly
* IPv4 and IPv6 support
-* Custom output handlers
+* Multiple user-selectable output formats and the ability to create custom output handlers
* Chainable plugins
-
+* Parallel processing option to divide the handling of data source into separate Python processes
+* Enables development of external plugin packs to share and install new externally developed plugins without overlapping the core Dshell plugin directories
+
+## Guides
+* [Dshell User Guide](Dshell_User_Guide.pdf)
+ * A guide to installation as well as both basic and advanced analysis with examples
+ * Helps new and experienced end users with using and understanding the decoder-shell (Dshell) framework
+* [Dshell Developer Guide](Dshell_Developer_Guide.pdf)
+ * A guide to plugin development with basic examples, as well as core function and class definitions, and an overview of data flow
+ * Helps end users develop new, custom Dshell plugins as well as modify existing plugins
+
## Requirements
-* Linux (developed on Red Hat Enterprise Linux 6.7)
-* Python 3 (developed with Python 3.6.2)
+* Linux (developed on Ubuntu 20.04 LTS)
+* Python 3 (developed with Python 3.8.10)
* [pypacker](https://gitlab.com/mike01/pypacker)
-* [pcapy](https://github.com/helpsystems/pcapy)
+* [pcapy-ng](https://github.com/stamparm/pcapy-ng/)
* [pyOpenSSL](https://github.com/pyca/pyopenssl)
* [geoip2](https://github.com/maxmind/GeoIP2-python)
- * [MaxMind GeoIP2 datasets](https://dev.maxmind.com/geoip/geoip2/geolite2/)
+ * [MaxMind GeoIP2 data sets](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data)
+ * Used to map IP addresses to country codes
+ * See Installation section for configuration
## Optional
* [oui.txt](http://standards-oui.ieee.org/oui.txt)
@@ -27,37 +39,11 @@ Key features:
* [pyJA3](https://github.com/salesforce/ja3/tree/master/python)
* used in the tls plugin
-## Major Changes Since Previous Release
-* This is a major framework update to Dshell. Plugins written for the previous version are not compatible with this version, and vice versa.
-* Uses Python 3
- * Rewritten in Python 3 from the ground up. Python 2 language deprecated on [1 JAN 2020](https://www.python.org/doc/sunset-python-2/)
- * By extension, dpkt and pypcap have been replaced with Python3-friendly pypacker and pcapy (respectively).
-* Is a Python package
- * Converted into a single package, removing the need for the shell to set several environment variables.
- * Allows easier use of Dshell plugins in other Python scripts
-* Changed "decoders" to "plugins"
- * Primarily a word-swap, to clarify that "decoders" can do more than simply decode traffic, and to put Dshell more in line with the terminology of other frameworks.
-* Significant reduction in camelCase functions, replaced with more Pythonic snake\_case functions.
- * Notable examples include blobHandler->blob\_handler, rawHandler->raw\_handler, connectionInitHandler->connection\_init\_handler, etc.
-* All plugins are now chainable
- * To accommodate this, handler functions in plugins must now use return statements indicating whether a packet, connection, or similar will continue to the next plugin. The type of object(s) to return depends on the type of handler, but will generally match the types of the handler's input. Dshell will display a warning if it's not the right type.
-* Plugins can now use all output modules\* available to the command line switch, -O
- * That does not mean every output module will be _useful_ to every plugin (e.g. using netflow output for a plugin that looks at individual packets), but they are available.
- * alert(), write(), and dump() are now the same function: write()
- * Output modules can be listed with a new flag in decode.py, --list-output or --lo
- * Arguments for output modules are now passed with the --oargs command-line argument
- * \* pcapout is (currently) the exception to this rule. A method has yet to arise that allows it to work with connection-based plugins
-* No more dObj declaration
- * decode.py just looks for the class named DshellPlugin and creates an instance of that
-* Improved error handling
- * Dshell handles more of the most common exceptions during everyday use
-* Enables development of external plugin packs, allowing the sharing and installation of new, externally-developed plugins without overlapping the core Dshell libraries.
-
## Installation
1. Install Dshell with pip
- * `sudo python3 -m pip install Dshell/` OR `sudo python3 -m pip install `
-2. Configure geoip2 by moving the MaxMind data files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) to <install-location>/data/GeoIP/
+ * `python3 -m pip install Dshell/` OR `python3 -m pip install `
+2. Configure geoip2 by placing the MaxMind GeoLite2 data set files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) in [...]/site-packages/dshell/data/GeoIP/
3. Run `dshell`. This should drop you into a `Dshell> ` prompt.
## Basic Usage
@@ -65,11 +51,11 @@ Key features:
* `decode -l`
* This will list all available plugins, alongside basic information about them
* `decode -h`
- * Show generic command-line flags available to most plugins
+ * Show generic command-line flags available to most plugins, such as the color blind friendly mode for all color output
* `decode -p `
* Display information about a plugin, including available command line flags
* `decode -p `
- * Run the selected plugin on a pcap file
+ * Run the selected plugin on a pcap or pcapng file
* `decode -p + `
* Chain two (or more) plugins together and run them on a pcap file
* `decode -p -i `
@@ -79,7 +65,7 @@ Key features:
Showing DNS lookups in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted)
```
-Dshell> decode -p dns ~/pcap/dns.cap |sort
+Dshell> decode -p dns ~/pcap/dns.cap | sort
[DNS] 2005-03-30 03:47:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 4146, TXT? google.com., TXT: b'\x0fv=spf1 ptr ?all' **
[DNS] 2005-03-30 03:47:50 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 63343, MX? google.com., MX: b'\x00(\x05smtp4\xc0\x0c', MX: b'\x00\n\x05smtp5\xc0\x0c', MX: b'\x00\n\x05smtp6\xc0\x0c', MX: b'\x00\n\x05smtp1\xc0\x0c', MX: b'\x00\n\x05smtp2\xc0\x0c', MX: b'\x00(\x05smtp3\xc0\x0c' **
[DNS] 2005-03-30 03:47:59 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 18849, LOC? google.com. **
@@ -165,8 +151,8 @@ Dshell> decode -p country+netflow --country_code=JP ~/pcap/SkypeIRC.cap
Collecting DNS traffic from several files and storing it in a new pcap file.
```
-Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap >/dev/null
-Dshell> tcpdump -nnr test.pcap |head
+Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap > /dev/null
+Dshell> tcpdump -nnr test.pcap | head
reading from file test.pcap, link-type EN10MB (Ethernet)
15:36:08.670569 IP 192.168.1.2.2131 > 192.168.1.1.53: 40209+ A? ui.skype.com. (30)
15:36:08.670687 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30)
@@ -184,8 +170,8 @@ Collecting TFTP data and converting alerts to JSON format using [sample traffic]
```
Dshell> decode -p tftp -O jsonout ~/pcap/tftp_*.pcap
-{"dport": 3445, "dip": "192.168.0.10", "data": "read rfc1350.txt (24599 bytes) ", "sport": 50618, "readwrite": "read", "sip": "192.168.0.253", "plugin": "tftp", "ts": 1367411051.972852, "filename": "rfc1350.txt"}
-{"dport": 2087, "dip": "192.168.0.13", "data": "write rfc1350.txt (24599 bytes) ", "sport": 57509, "readwrite": "write", "sip": "192.168.0.1", "plugin": "tftp", "ts": 1367053679.45274, "filename": "rfc1350.txt"}
+{"ts": 1367411051.972852, "sip": "192.168.0.253", "sport": 50618, "dip": "192.168.0.10", "dport": 3445, "readwrite": "read", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_rrq.pcap", "data": "read rfc1350.txt (24599 bytes) "}
+{"ts": 1367053679.45274, "sip": "192.168.0.1", "sport": 57509, "dip": "192.168.0.13", "dport": 2087, "readwrite": "write", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_wrq.pcap", "data": "write rfc1350.txt (24599 bytes) "}
```
Running a plugin within a separate Python script using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP)
@@ -198,7 +184,7 @@ import dshell.plugins.tftp.tftp as tftp
# Instantiate plugin
plugin = tftp.DshellPlugin()
# Define plugin-specific arguments, if needed
-dargs = {plugin: {"outdir": "/tmp/"}}
+dargs = {plugin: {"rip": True, "outdir": "/tmp/"}}
# Add plugin(s) to plugin chain
decode.plugin_chain = [plugin]
# Run decode main function with all other arguments
diff --git a/README.md b/README.md
index 10a2f6a..e1e0f6f 100644
--- a/README.md
+++ b/README.md
@@ -5,17 +5,29 @@ Key features:
* Deep packet analysis using specialized plugins
* Robust stream reassembly
* IPv4 and IPv6 support
-* Custom output handlers
+* Multiple user-selectable output formats and the ability to create custom output handlers
* Chainable plugins
+* Parallel processing option to divide the handling of data source into separate Python processes
+* Enables development of external plugin packs to share and install new externally developed plugins without overlapping the core Dshell plugin directories
+
+## Guides
+* [Dshell User Guide](Dshell_User_Guide.pdf)
+ * A guide to installation as well as both basic and advanced analysis with examples
+ * Helps new and experienced end users with using and understanding the decoder-shell (Dshell) framework
+* [Dshell Developer Guide](Dshell_Developer_Guide.pdf)
+ * A guide to plugin development with basic examples, as well as core function and class definitions, and an overview of data flow
+ * Helps end users develop new, custom Dshell plugins as well as modify existing plugins
## Requirements
-* Linux (developed on Red Hat Enterprise Linux 6.7)
-* Python 3 (developed with Python 3.6.2)
+* Linux (developed on Ubuntu 20.04 LTS)
+* Python 3 (developed with Python 3.8.10)
* [pypacker](https://gitlab.com/mike01/pypacker)
-* [pcapy](https://github.com/helpsystems/pcapy)
+* [pcapy-ng](https://github.com/stamparm/pcapy-ng/)
* [pyOpenSSL](https://github.com/pyca/pyopenssl)
* [geoip2](https://github.com/maxmind/GeoIP2-python)
- * [MaxMind GeoIP2 datasets](https://dev.maxmind.com/geoip/geoip2/geolite2/)
+ * [MaxMind GeoIP2 data sets](https://dev.maxmind.com/geoip/geolite2-free-geolocation-data)
+ * Used to map IP addresses to country codes
+ * See Installation section for configuration
## Optional
* [oui.txt](http://standards-oui.ieee.org/oui.txt)
@@ -27,37 +39,11 @@ Key features:
* [pyJA3](https://github.com/salesforce/ja3/tree/master/python)
* used in the tls plugin
-## Major Changes Since Previous Release
-* This is a major framework update to Dshell. Plugins written for the previous version are not compatible with this version, and vice versa.
-* Uses Python 3
- * Rewritten in Python 3 from the ground up. Python 2 language deprecated on [1 JAN 2020](https://www.python.org/doc/sunset-python-2/)
- * By extension, dpkt and pypcap have been replaced with Python3-friendly pypacker and pcapy (respectively).
-* Is a Python package
- * Converted into a single package, removing the need for the shell to set several environment variables.
- * Allows easier use of Dshell plugins in other Python scripts
-* Changed "decoders" to "plugins"
- * Primarily a word-swap, to clarify that "decoders" can do more than simply decode traffic, and to put Dshell more in line with the terminology of other frameworks.
-* Significant reduction in camelCase functions, replaced with more Pythonic snake\_case functions.
- * Notable examples include blobHandler->blob\_handler, rawHandler->raw\_handler, connectionInitHandler->connection\_init\_handler, etc.
-* All plugins are now chainable
- * To accommodate this, handler functions in plugins must now use return statements indicating whether a packet, connection, or similar will continue to the next plugin. The type of object(s) to return depends on the type of handler, but will generally match the types of the handler's input. Dshell will display a warning if it's not the right type.
-* Plugins can now use all output modules\* available to the command line switch, -O
- * That does not mean every output module will be _useful_ to every plugin (e.g. using netflow output for a plugin that looks at individual packets), but they are available.
- * alert(), write(), and dump() are now the same function: write()
- * Output modules can be listed with a new flag in decode.py, --list-output or --lo
- * Arguments for output modules are now passed with the --oargs command-line argument
- * \* pcapout is (currently) the exception to this rule. A method has yet to arise that allows it to work with connection-based plugins
-* No more dObj declaration
- * decode.py just looks for the class named DshellPlugin and creates an instance of that
-* Improved error handling
- * Dshell handles more of the most common exceptions during everyday use
-* Enables development of external plugin packs, allowing the sharing and installation of new, externally-developed plugins without overlapping the core Dshell libraries.
-
## Installation
1. Install Dshell with pip
- * `sudo python3 -m pip install Dshell/` OR `sudo python3 -m pip install `
-2. Configure geoip2 by moving the MaxMind data files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) to <install-location>/data/GeoIP/
+ * `python3 -m pip install Dshell/` OR `python3 -m pip install `
+2. Configure geoip2 by placing the MaxMind GeoLite2 data set files (GeoLite2-ASN.mmdb, GeoLite2-City.mmdb, GeoLite2-Country.mmdb) in [...]/site-packages/dshell/data/GeoIP/
3. Run `dshell`. This should drop you into a `Dshell> ` prompt.
## Basic Usage
@@ -65,11 +51,11 @@ Key features:
* `decode -l`
* This will list all available plugins, alongside basic information about them
* `decode -h`
- * Show generic command-line flags available to most plugins
+ * Show generic command-line flags available to most plugins, such as the color blind friendly mode for all color output
* `decode -p `
* Display information about a plugin, including available command line flags
* `decode -p `
- * Run the selected plugin on a pcap file
+ * Run the selected plugin on a pcap or pcapng file
* `decode -p + `
* Chain two (or more) plugins together and run them on a pcap file
* `decode -p -i `
@@ -79,7 +65,7 @@ Key features:
Showing DNS lookups in [sample traffic](http://wiki.wireshark.org/SampleCaptures#General_.2F_Unsorted)
```
-Dshell> decode -p dns ~/pcap/dns.cap |sort
+Dshell> decode -p dns ~/pcap/dns.cap | sort
[DNS] 2005-03-30 03:47:46 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 4146, TXT? google.com., TXT: b'\x0fv=spf1 ptr ?all' **
[DNS] 2005-03-30 03:47:50 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 63343, MX? google.com., MX: b'\x00(\x05smtp4\xc0\x0c', MX: b'\x00\n\x05smtp5\xc0\x0c', MX: b'\x00\n\x05smtp6\xc0\x0c', MX: b'\x00\n\x05smtp1\xc0\x0c', MX: b'\x00\n\x05smtp2\xc0\x0c', MX: b'\x00(\x05smtp3\xc0\x0c' **
[DNS] 2005-03-30 03:47:59 192.168.170.8:32795 -- 192.168.170.20:53 ** ID: 18849, LOC? google.com. **
@@ -165,8 +151,8 @@ Dshell> decode -p country+netflow --country_code=JP ~/pcap/SkypeIRC.cap
Collecting DNS traffic from several files and storing it in a new pcap file.
```
-Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap >/dev/null
-Dshell> tcpdump -nnr test.pcap |head
+Dshell> decode -p dns+pcapwriter --pcapwriter_outfile=test.pcap ~/pcap/*.cap > /dev/null
+Dshell> tcpdump -nnr test.pcap | head
reading from file test.pcap, link-type EN10MB (Ethernet)
15:36:08.670569 IP 192.168.1.2.2131 > 192.168.1.1.53: 40209+ A? ui.skype.com. (30)
15:36:08.670687 IP 192.168.1.2.2131 > 192.168.1.1.53: 40210+ AAAA? ui.skype.com. (30)
@@ -184,8 +170,8 @@ Collecting TFTP data and converting alerts to JSON format using [sample traffic]
```
Dshell> decode -p tftp -O jsonout ~/pcap/tftp_*.pcap
-{"dport": 3445, "dip": "192.168.0.10", "data": "read rfc1350.txt (24599 bytes) ", "sport": 50618, "readwrite": "read", "sip": "192.168.0.253", "plugin": "tftp", "ts": 1367411051.972852, "filename": "rfc1350.txt"}
-{"dport": 2087, "dip": "192.168.0.13", "data": "write rfc1350.txt (24599 bytes) ", "sport": 57509, "readwrite": "write", "sip": "192.168.0.1", "plugin": "tftp", "ts": 1367053679.45274, "filename": "rfc1350.txt"}
+{"ts": 1367411051.972852, "sip": "192.168.0.253", "sport": 50618, "dip": "192.168.0.10", "dport": 3445, "readwrite": "read", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_rrq.pcap", "data": "read rfc1350.txt (24599 bytes) "}
+{"ts": 1367053679.45274, "sip": "192.168.0.1", "sport": 57509, "dip": "192.168.0.13", "dport": 2087, "readwrite": "write", "filename": "rfc1350.txt", "plugin": "tftp", "pcapfile": "/home/pcap/tftp_wrq.pcap", "data": "write rfc1350.txt (24599 bytes) "}
```
Running a plugin within a separate Python script using [sample traffic](https://wiki.wireshark.org/SampleCaptures#TFTP)
@@ -198,7 +184,7 @@ import dshell.plugins.tftp.tftp as tftp
# Instantiate plugin
plugin = tftp.DshellPlugin()
# Define plugin-specific arguments, if needed
-dargs = {plugin: {"outdir": "/tmp/"}}
+dargs = {plugin: {"rip": True, "outdir": "/tmp/"}}
# Add plugin(s) to plugin chain
decode.plugin_chain = [plugin]
# Run decode main function with all other arguments
diff --git a/dshell/core.py b/dshell/core.py
index 8452036..9b57fd9 100644
--- a/dshell/core.py
+++ b/dshell/core.py
@@ -38,7 +38,7 @@
logger = logging.getLogger(__name__)
-__version__ = "3.2.1"
+__version__ = "3.2.3"
class SequenceNumberError(Exception):
"""
@@ -258,22 +258,43 @@ def recompile_bpf(self):
else:
raise e
- def ipdefrag(self, pkt):
+ def ipdefrag(self, packet: 'Packet') -> 'Packet':
"""
IP fragment reassembly
- """
- if isinstance(pkt, ip.IP): # IPv4
- f = self._packet_fragments[(pkt.src, pkt.dst, pkt.id)]
- f[pkt.offset] = pkt
- if not pkt.flags & 0x1:
+ Store the first seen packet, collect data from followup packets, then
+ glue it all together and update that first packet with new data
+ """
+ pkt = packet.pkt
+ ipp = pkt.upper_layer
+ if isinstance(ipp, ip.IP): # IPv4
+ f = self._packet_fragments[(ipp.src, ipp.dst, ipp.id)]
+ f[ipp.offset] = packet
+
+ if not ipp.flags & 0x1: # If no more fragments (MF)
+ if len(f) <= 1 and 0 in f:
+ # If only one unfragmented packet, return that packet
+ del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)]
+ return f[0]
+ elif 0 not in f:
+ logger.debug(f"Missing first fragment of fragmented packet. Dropping ({packet.sip} -> {packet.dip}: {ipp.id}:{ipp.flags}:{ipp.offset})")
+ del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)]
+ return None
+ fkeys = sorted(f.keys())
data = b''
- for key in sorted(f.keys()):
- data += f[key].body_bytes
- del self._packet_fragments[(pkt.src, pkt.dst, pkt.id)]
- newpkt = ip.IP(pkt.header_bytes + data)
- newpkt.bin(update_auto_fields=True) # refresh checksum
- return newpkt
+ firstpacket = f[fkeys[0]]
+ for key in fkeys:
+ data += f[key].pkt.upper_layer.body_bytes
+ newip = ip.IP(firstpacket.pkt.upper_layer.header_bytes + data)
+ newip.bin(update_auto_fields=True) # refresh checksum
+ firstpacket.pkt.upper_layer = newip
+ del self._packet_fragments[(ipp.src, ipp.dst, ipp.id)]
+ return Packet(
+ firstpacket.pkt.__len__,
+ firstpacket.pkt,
+ firstpacket.ts,
+ firstpacket.frame
+ )
elif isinstance(pkt, ip6.IP6): # IPv6
# TODO handle IPv6 offsets https://en.wikipedia.org/wiki/IPv6_packet#Fragment
@@ -398,16 +419,14 @@ def consume_packet(self, packet: "Packet"):
self.seen_packet_count.value += 1
# Attempt to perform defragmentation
- if isinstance(packet.pkt.upper_layer, (ip.IP, ip6.IP6)):
- ipp = packet.pkt.upper_layer
- if self.defrag_ip:
- ipp = self.ipdefrag(ipp)
- if not ipp:
- # we do not yet have all of the packet fragments, so move
- # on to next packet for now
- return
- else:
- packet.pkt.upper_layer = ipp
+ if self.defrag_ip and isinstance(packet.pkt.upper_layer, (ip.IP, ip6.IP6)):
+ defragpkt = self.ipdefrag(packet)
+ if not defragpkt:
+ # we do not yet have all of the packet fragments, so move
+ # on to next packet for now
+ return
+ else:
+ packet = defragpkt
# call packet_handler and return its output
# decode.py will continue down the chain if it returns anything
@@ -494,7 +513,7 @@ def __init__(self, **kwargs):
# timestamp of the current packet, minus this value
self.timeout = datetime.timedelta(hours=1)
# The number of packets to process between timeout checks.
- self.timeout_frequency = 300
+ self.timeout_frequency = 50
# The maximum number of open connections allowed at one time.
# If the maximum number of connections is met, the oldest connections
# will be force closed.
@@ -577,12 +596,12 @@ def _connection_handler(self, packet: "Packet"):
connection should close.
"""
# Sort the addr value for consistent dictionary key purposes
- addr = tuple(sorted(packet.addr))
+ connkey = tuple(sorted(packet.addr) + [packet.protocol_num])
# If this is a new connection, initialize it and call the init handler
- if addr not in self._connection_tracker:
+ if connkey not in self._connection_tracker:
conn = Connection(packet)
- self._connection_tracker[addr] = conn
+ self._connection_tracker[connkey] = conn
try:
self.connection_init_handler(conn)
except Exception as e:
@@ -591,7 +610,7 @@ def _connection_handler(self, packet: "Packet"):
with self.seen_conn_count.get_lock():
self.seen_conn_count.value += 1
else:
- conn = self._connection_tracker[addr]
+ conn = self._connection_tracker[connkey]
conn.add_packet(packet)
# TODO: Do we need this? This flag is set to False when the connection is initialized and not
@@ -628,7 +647,8 @@ def _close_connection(self, conn, full=False):
# Remove connection from tracker once in the queue.
try:
- del self._connection_tracker[tuple(sorted(conn.addr))]
+ connkey = tuple(sorted(conn.addr) + [conn.protocol_num])
+ del self._connection_tracker[connkey]
except KeyError:
pass
@@ -802,6 +822,7 @@ def __init__(self, plugin, pktlen, pkt, ts):
Attributes:
ts: timestamp of packet
dt: datetime of packet
+ frame: sequential packet number as read from data stream
pkt: pypacker object for the packet
rawpkt: raw bytestring of the packet
pktlen: length of packet
@@ -881,6 +902,13 @@ def __init__(self, pktlen, packet: pypacker.Packet, timestamp: int, frame=0):
ieee80211_p = layer
elif ip_p is None and isinstance(layer, (ip.IP, ip6.IP6)):
ip_p = layer
+ try:
+ if ip_p.flags & 0x1 and ip_p.offset > 0:
+ # IP fragmentation, break all further layer processing
+ break
+ except AttributeError:
+ # IPv6 does not always have flags header field set
+ pass
elif tcp_p is None and isinstance(layer, tcp.TCP):
tcp_p = layer
elif udp_p is None and isinstance(layer, udp.UDP):
@@ -1052,6 +1080,7 @@ def info(self):
"""
d = {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
d['byte_count'] = self.byte_count
+ d['rawpkt'] = self.pkt.bin()
del d['pkt']
return d
@@ -1096,6 +1125,7 @@ def __init__(self, plugin, first_packet)
serverlon: same as diplon
serverasn: same as dipasn
protocol: text version of protocol in layer-3 header
+ protocol_num: numeric version of protocol in layer-3 header
clientpackets: counts of packets from client side
clientbytes: total bytes transferred from client side
serverpackets: counts of packets from server side
@@ -1161,6 +1191,7 @@ def __init__(self, first_packet):
self.serverlon = first_packet.diplon
self.serverasn = first_packet.dipasn
self.protocol = first_packet.protocol
+ self.protocol_num = first_packet.protocol_num
self.ts = first_packet.ts
self.dt = first_packet.dt
self.starttime = first_packet.dt
@@ -1172,6 +1203,9 @@ def __init__(self, first_packet):
self.stop = False
self.handled = False
+ # Cache of created blobs
+ self._blob_cache = []
+
self.add_packet(first_packet)
@property
@@ -1197,37 +1231,42 @@ def blobs(self) -> Iterable["Blob"]:
This is dynamically generated on-demand based on the current set of packets in the connection.
"""
- blobs = []
+ if self._blob_cache:
+ yield from self._blob_cache
- for packet in self.packets:
- # TODO: skipping packets without data greatly improves speed, but we may want to
- # allow them if we support using ack numbers.
- if not packet.data:
- continue
+ else:
+ blobs = []
- # If we see a sequence for an old blob, this is a retransmission.
- # Find the blob and add this packet.
- # NOTE: There is probably more to it than this, but this seems to work for now.
- seq = packet.sequence_number
- if seq is not None:
- found = False
- for blob in blobs:
- if blob.sip == packet.sip and seq in blob.sequence_range:
- blob.add_packet(packet)
- found = True
- break
- if found:
+ for packet in self.packets:
+ # TODO: skipping packets without data greatly improves speed, but we may want to
+ # allow them if we support using ack numbers.
+ if not packet.data:
continue
- # Create a new message if the first or the other direction has started sending data.
- if not blobs or (packet.sip != blobs[-1].sip and packet.data):
- blobs.append(Blob(self, packet))
-
- # Otherwise add packet to last blob.
- else:
- blobs[-1].add_packet(packet)
+ # If we see a sequence for an old blob, this is a retransmission.
+ # Find the blob and add this packet.
+ # NOTE: There is probably more to it than this, but this seems to work for now.
+ seq = packet.sequence_number
+ if seq is not None:
+ found = False
+ for blob in blobs:
+ if blob.sip == packet.sip and seq in blob.sequence_range:
+ blob.add_packet(packet)
+ found = True
+ break
+ if found:
+ continue
+
+ # Create a new message if the first or the other direction has started sending data.
+ if not blobs or (packet.sip != blobs[-1].sip and packet.data):
+ blobs.append(Blob(self, packet))
+
+ # Otherwise add packet to last blob.
+ else:
+ blobs[-1].add_packet(packet)
- yield from blobs
+ self._blob_cache = blobs
+ yield from blobs
def add_packet(self, packet: Packet):
"""
@@ -1239,6 +1278,8 @@ def add_packet(self, packet: Packet):
raise ValueError(f"Address {repr(packet.sip)} is not part of connection.")
self.packets.append(packet)
+ # A new packet means we might need to recalculate all of the blobs
+ self._blob_cache = []
# Adjust state if packet is part of a startup or shutdown.
if packet.tcp_flags is not None:
@@ -1273,11 +1314,18 @@ def info(self):
Dictionary with information
"""
d = {k: v for k, v in self.__dict__.items() if not k.startswith('_')}
+
+ cb, cp, sb, sp = self.bytes_and_counts()
+
d['duration'] = self.duration
- d['clientbytes'] = self.clientbytes
- d['clientpackets'] = self.clientpackets
- d['serverbytes'] = self.serverbytes
- d['serverpackets'] = self.serverpackets
+# d['clientbytes'] = self.clientbytes
+# d['clientpackets'] = self.clientpackets
+# d['serverbytes'] = self.serverbytes
+# d['serverpackets'] = self.serverpackets
+ d['clientbytes'] = cb
+ d['clientpackets'] = cp
+ d['serverbytes'] = sb
+ d['serverpackets'] = sp
del d['stop']
del d['handled']
del d['packets']
@@ -1293,10 +1341,36 @@ def _server_packets(self) -> Iterable[Packet]:
if packet.addr != self.addr:
yield packet
+ def bytes_and_counts(self) -> Tuple[int, int, int, int]:
+ """
+ Convenience function to get client and server packet and byte counts
+ while only iterating over the packet list once.
+ Returns a tuple of:
+ (client bytes, client packets, server bytes, server packets)
+ """
+ cbytes, cpkts, sbytes, spkts = 0, 0, 0, 0
+ for packet in self.packets:
+ if packet.addr == self.addr:
+ # client
+ cbytes += packet.byte_count
+ cpkts += bool(packet.byte_count) # only count packets with data
+ else:
+ # server
+ sbytes += packet.byte_count
+ spkts += bool(packet.byte_count) # only count packets with data
+ return (cbytes, cpkts, sbytes, spkts)
+
+ @property
+ def totalbytes(self) -> int:
+ """
+ The total number of bytes from both directions
+ """
+ return sum(packet.byte_count for packet in self.packets)
+
@property
def clientbytes(self) -> int:
"""
- The total number of bytes form the client.
+ The total number of bytes from the client.
"""
return sum(packet.byte_count for packet in self._client_packets())
@@ -1311,7 +1385,7 @@ def clientpackets(self) -> int:
@property
def serverbytes(self) -> int:
"""
- The total number of bytes form the server.
+ The total number of bytes from the server.
"""
return sum(packet.byte_count for packet in self._server_packets())
@@ -1324,6 +1398,7 @@ def serverpackets(self) -> int:
return sum(bool(packet.byte_count) for packet in self._server_packets())
def __repr__(self):
+ cb, cp, sb, sp = self.bytes_and_counts()
return '%s %16s -> %16s (%s -> %s) %6s %6s %5d %5d %7d %7d %-.4fs' % (
self.starttime,
self.clientip,
@@ -1332,10 +1407,14 @@ def __repr__(self):
self.servercc,
self.clientport,
self.serverport,
- self.clientpackets,
- self.serverpackets,
- self.clientbytes,
- self.serverbytes,
+# self.clientpackets,
+# self.serverpackets,
+# self.clientbytes,
+# self.serverbytes,
+ cp,
+ sp,
+ cb,
+ sb,
self.duration,
)
@@ -1391,6 +1470,8 @@ def __init__(self, connection: Connection, first_packet):
self.connection = connection
self.addr = first_packet.addr
self.ts = first_packet.ts
+ self.starttime = first_packet.ts
+ self.endtime = first_packet.ts
self.sip = first_packet.sip
self.smac = first_packet.smac
self.sport = first_packet.sport
@@ -1413,6 +1494,8 @@ def __init__(self, connection: Connection, first_packet):
# Maps sequence number with packets
self._seq_map = {}
+ self.seq_max = 0
+ self.seq_min = 0
# Used to indicate that a Blob should not be passed to next plugin.
# Can theoretically be overruled in, say, a connection_handler to
@@ -1434,17 +1517,17 @@ def all_packets(self):
warnings.warn("all_packets has been replaced with packets attribute", DeprecationWarning)
return self.packets
- @property
- def starttime(self):
- return min(packet.dt for packet in self.packets)
+# @property
+# def starttime(self):
+# return min(packet.dt for packet in self.packets)
@property
def start_time(self):
return self.starttime
- @property
- def endtime(self):
- return max(packet.dt for packet in self.packets)
+# @property
+# def endtime(self):
+# return max(packet.dt for packet in self.packets)
@property
def end_time(self):
@@ -1515,13 +1598,17 @@ def sequence_range(self) -> range:
"""
The range of sequence numbers found within the packets.
"""
- sequence_numbers = self.sequence_numbers
- if not sequence_numbers:
+# sequence_numbers = self.sequence_numbers
+# if not sequence_numbers:
+# return range(0, 0)
+#
+# min_seq = min(sequence_numbers)
+# max_seq = max(sequence_numbers)
+# return range(min_seq, max_seq + len(self._seq_map[max_seq].data))
+ if not self._seq_map:
return range(0, 0)
- min_seq = min(sequence_numbers)
- max_seq = max(sequence_numbers)
- return range(min_seq, max_seq + len(self._seq_map[max_seq].data))
+ return range(self.seq_min, self.seq_max + len(self._seq_map[self.seq_max].data))
@property
def segments(self) -> List[Tuple[int, "Packet"]]:
@@ -1546,7 +1633,7 @@ def segments(self) -> List[Tuple[int, "Packet"]]:
if missing_num_bytes:
logger.debug(
f"Missing {missing_num_bytes} bytes of data between packets "
- f"{prev_packet and prev_packet.frame} and {packet.frame}"
+ f"{prev_packet.frame} and {packet.frame}"
)
expected_seq += missing_num_bytes + len(packet.data)
prev_packet = packet
@@ -1783,7 +1870,7 @@ def reassemble(self, allow_padding=True, allow_overlap=True, padding=b'\x00'):
def info(self):
"""
Provides a dictionary with information about a blob. Useful for
- calls to a plugin's write() function, e.g. self.write(\\*\\*conn.info())
+ calls to a plugin's write() function, e.g. self.write(\\*\\*blob.info())
Returns:
Dictionary with information
@@ -1815,11 +1902,17 @@ def add_packet(self, packet):
# If packet is not TCP just add packet to list.
if seq is None:
self.packets.append(packet)
+ if packet.ts < self.starttime: self.starttime = packet.ts
+ if packet.ts > self.endtime: self.endtime = packet.ts
return
# If this a new sequence number we haven't seen before, add it to the map.
if seq not in self._seq_map:
self._seq_map[seq] = packet
+ if seq < self.seq_min: self.seq_min = seq
+ if seq > self.seq_max: self.seq_max = seq
+ if packet.ts < self.starttime: self.starttime = packet.ts
+ if packet.ts > self.endtime: self.endtime = packet.ts
self.packets.append(packet)
return
@@ -1829,7 +1922,8 @@ def add_packet(self, packet):
orig_packet = self._seq_map[seq]
# ignore duplicate packet.
- if len(packet.data) <= len(orig_packet.data):
+# if len(packet.data) <= len(orig_packet.data):
+ if packet.data == orig_packet.data:
# TODO: should we still handle duplicate packets.
logger.debug(f'Ignoring duplicate packet: {packet.frame}')
return
@@ -1840,7 +1934,8 @@ def add_packet(self, packet):
orig_next_seq = seq + len(orig_packet.data)
next_seq = seq + len(packet.data)
if (
- next_seq < max(self.sequence_numbers)
+# next_seq < max(self.sequence_numbers)
+ next_seq < self.seq_max
and orig_packet.data
and next_seq not in self._seq_map
and orig_next_seq in self._seq_map
@@ -1855,6 +1950,8 @@ def add_packet(self, packet):
logger.debug(f'Replacing packet {orig_packet.frame} with {packet.frame}')
self._seq_map[seq] = packet
self.packets = [packet if p.sequence_number == seq else p for p in self.packets]
+ if packet.ts < self.starttime: self.starttime = packet.ts
+ if packet.ts > self.endtime: self.endtime = packet.ts
# Now remove any packets that contained data that is now part of the retransmitted packet.
packets_to_remove = []
@@ -1877,5 +1974,7 @@ def _remove_packet(self, packet):
for seq, packet_ in list(self._seq_map.items()):
if packet_ == packet:
del self._seq_map[seq]
+ if seq == self.seq_max:
+ self.seq_max = max(self._seq_map.keys())
self.packets.remove(packet)
diff --git a/dshell/decode.py b/dshell/decode.py
index 9d01922..8b37be1 100755
--- a/dshell/decode.py
+++ b/dshell/decode.py
@@ -32,6 +32,7 @@
import tempfile
import zipfile
from collections import OrderedDict
+from datetime import timedelta
from getpass import getpass
from glob import glob
from importlib import import_module
@@ -232,26 +233,38 @@ def main(plugin_args=None, **kwargs):
for plugin in plugin_chain:
plugin.out.set_oargs(**oargs)
- # If writing to a file, set for each output module here
- if kwargs.get("outfile", None):
- for plugin in plugin_chain:
+ for plugin in plugin_chain:
+ # If writing to a file, set for each output module here
+ if kwargs.get("outfile", None):
plugin.out.reset_fh(filename=kwargs["outfile"])
- # Set nobuffer mode if that's what the user wants
- if kwargs.get("nobuffer", False):
- for plugin in plugin_chain:
+ # Set nobuffer mode if that's what the user wants
+ if kwargs.get("nobuffer", False):
plugin.out.nobuffer = True
+
+ # Set color blind friendly mode
+ if kwargs.get("cbf", False):
+ plugin.out.cbf = True
- # Set the extra flag for all output modules
- if kwargs.get("extra", False):
- for plugin in plugin_chain:
+ # Set the extra flag for all output modules
+ if kwargs.get("extra", False):
plugin.out.extra = True
plugin.out.set_format(plugin.out.format)
- # Set the BPF filters
- # Each plugin has its own default BPF that will be extended or replaced
- # based on --no-vlan, --ebpf, or --bpf arguments.
- for plugin in plugin_chain:
+ # Set some attributes for ConnectionPlugins
+ if hasattr(plugin, "timeout"):
+ # Set wait time since last packet arrived in a connection before
+ # considering connection closed
+ if t := kwargs.get("conntimeout"):
+ td = timedelta(seconds=int(t))
+ plugin.timeout = td
+ # Set max number of allowed open connections
+ if t := kwargs.get("connmax"):
+ plugin.max_open_connections = int(t)
+
+ # Set the BPF filters
+ # Each plugin has its own default BPF that will be extended or replaced
+ # based on --no-vlan, --ebpf, or --bpf arguments.
if kwargs.get("bpf", None):
plugin.bpf = kwargs.get("bpf", "")
continue
@@ -390,7 +403,7 @@ def read_packets(input: str, interface=False, bpf=None, count=None) -> Iterable[
if interface:
# Listen on an interface if the option is set
try:
- capture = pcapy.open_live(input, 65536, True, 0)
+ capture = pcapy.open_live(input, 65536, True, 1)
except pcapy.PcapError as e:
# User probably doesn't have permission to listen on interface
# In any case, print just the error without traceback
@@ -446,8 +459,12 @@ def read_packets(input: str, interface=False, bpf=None, count=None) -> Iterable[
try:
header, packet_data = capture.next()
if header is None and not packet_data:
- # probably the end of the capture
- break
+ if not interface:
+ # probably the end of the capture
+ break
+ else:
+ # interface timed out
+ continue
if count and frame - 1 >= count:
# we've reached the maximum number of packets to process
break
@@ -463,6 +480,11 @@ def read_packets(input: str, interface=False, bpf=None, count=None) -> Iterable[
yield packet
+ # handle SIGINT gracefully, break read loop and allow shutdown
+ except KeyboardInterrupt:
+ logger.debug("Caught KeyboardInterrupt or SIGINT. Closing capture.")
+ break
+
except pcapy.PcapError as e:
estr = str(e)
eformat = "Error processing '{i}' - {e}"
@@ -537,7 +559,7 @@ def main_command_line():
help="Show debug messages")
parser.add_argument('-v', '--verbose', action="store_true",
help="Show informational messages")
- parser.add_argument('-acc', '--allcc', action="store_true",
+ parser.add_argument('--acc', '--allcc', action="store_true",
help="Show all 3 GeoIP2 country code types (represented_country/registered_country/country)")
parser.add_argument('-d', '-p', '--plugin', dest='plugin', type=str,
action='append', metavar="PLUGIN",
@@ -556,6 +578,12 @@ def main_command_line():
parser.add_argument('--unzipdir', type=str, metavar="DIRECTORY",
default=tempfile.gettempdir(),
help='Directory to use when decompressing input files (.gz, .bz2, and .zip only)')
+ parser.add_argument('--conn-timeout', dest="conntimeout", type=int,
+ metavar="SECONDS", default=3600,
+ help="Number of seconds to wait after last packet in a connection before closing it (default: 3600)")
+ parser.add_argument('--conn-max-open', dest='connmax', type=int,
+ metavar="NUMBER", default=1000,
+ help="Number of connections to hold in an open state before Dshell begins closing the oldest (default: 1000)")
multiprocess_group = parser.add_argument_group("multiprocessing arguments")
multiprocess_group.add_argument('-P', '--parallel', dest='multiprocessing', action='store_true',
@@ -579,6 +607,9 @@ def main_command_line():
output_group.add_argument("--no-buffer", action="store_true",
help="Do not buffer plugin output",
dest="nobuffer")
+ output_group.add_argument("--cbf", "--color-blind-friendly", action="store_true",
+ help="Activate color blind friendly mode, colorout and htmlout output modules use yellow/gold in place of red and different shades of green/yellow/blue are used to help better differentiate between them",
+ dest="cbf")
output_group.add_argument("-x", "--extra", action="store_true",
help="Appends extra data to all plugin output.")
# TODO Figure out how to make --extra flag play nicely with user-only
@@ -619,6 +650,8 @@ def main_command_line():
help='List all available plugins', dest='list')
parser_short.add_argument("--lo", "--list-output", action="store_true",
help="List available output modules")
+ parser_short.add_argument("--cbf", "--color-blind-friendly", action="store_true",
+ help="Activate color blind friendly mode, colorout and htmlout output modules use yellow/gold in place of red and different shades of green/yellow/blue are used to help better differentiate between them")
# FIXME: Should this duplicate option be removed?
parser_short.add_argument("-o", "--omodule", type=str, metavar="MODULE",
help="Use specified output module for plugins instead of defaults. For example, --omodule=jsonout for JSON output.")
diff --git a/dshell/output/colorout.py b/dshell/output/colorout.py
index be25c07..76e1cea 100644
--- a/dshell/output/colorout.py
+++ b/dshell/output/colorout.py
@@ -37,10 +37,15 @@ def __init__(self, *args, **kwargs):
'cs': '31', # client-to-server is red
'sc': '32', # server-to-client is green
'--': '34', # everything else is blue
- } # TODO configurable for color-blind users?
+ }
self.hexmode = kwargs.get('hex', False)
self.format_is_set = False
+ def setup(self):
+ # activate color blind friendly mode
+ if self.cbf:
+ self.colors['cs'] = '33' #client-to-server is yellow
+
def write(self, *args, **kwargs):
if not self.format_is_set:
if 'clientip' in kwargs:
diff --git a/dshell/output/csvout.py b/dshell/output/csvout.py
index 8ba9cbd..0ad2b9c 100644
--- a/dshell/output/csvout.py
+++ b/dshell/output/csvout.py
@@ -15,32 +15,24 @@ class CSVOutput(Output):
A header row can be printed with --oarg header
Additional fields can be included with --oarg fields=field1,field2,field3
+ For example, MAC address can be included with --oarg fields=smac,dmac
+ Note: Field names must match the variable names in the plugin
- Note: Field names much match the variable names in the plugin
+ Additional flow fields for connection can be included with --oarg flows
"""
# TODO refine plugin to do things like wrap quotes around long strings
_DEFAULT_FIELDS = ['plugin', 'ts', 'sip', 'sport', 'dip', 'dport', 'data']
+ _DEFAULT_FLOW_FIELDS = ['plugin', 'starttime', 'clientip', 'serverip', 'clientcc', 'servercc', 'protocol', 'clientport', 'serverport', 'clientpackets', 'serverpackets', 'clientbytes', 'serverbytes', 'duration', 'data']
_DEFAULT_DELIM = ','
_DESCRIPTION = "CSV format output"
def __init__(self, *args, **kwargs):
- self.delimiter = kwargs.get('delimiter', self._DEFAULT_DELIM)
- if self.delimiter == 'tab':
- self.delimiter = '\t'
-
- self.use_header = kwargs.get("header", False)
-
+ self.use_header = False
self.fields = list(self._DEFAULT_FIELDS)
- exfields = kwargs.get("fields", "")
- for field in exfields.split(','):
- self.fields.append(field)
-
super().__init__(**kwargs)
- self.set_format()
-
def set_format(self, _=None):
"Set the format to a CSV list of fields"
columns = []
@@ -54,6 +46,12 @@ def set_format(self, _=None):
super().set_format(fmt)
def set_oargs(self, **kwargs):
+ self.use_header = kwargs.pop("header", False)
+ if kwargs.pop("flows", False):
+ self.fields = list(self._DEFAULT_FLOW_FIELDS)
+ if exfields := kwargs.pop("fields", None):
+ for field in exfields.split(','):
+ self.fields.append(field)
super().set_oargs(**kwargs)
self.set_format()
@@ -62,4 +60,4 @@ def setup(self):
self.fh.write(self.delimiter.join([f for f in self.fields]) + "\n")
-obj = CSVOutput
+obj = CSVOutput
\ No newline at end of file
diff --git a/dshell/output/htmlout.py b/dshell/output/htmlout.py
index 3deea20..0e0a1da 100644
--- a/dshell/output/htmlout.py
+++ b/dshell/output/htmlout.py
@@ -73,6 +73,10 @@ def __init__(self, *args, **kwargs):
self.format_is_set = False
def setup(self):
+ # activate color blind friendly mode
+ if self.cbf:
+ self.colors['cs'] = 'gold' # client-to-server is gold (darker yellow)
+ self.colors['sc'] = 'seagreen' # server-to-client is sea green (lighter green)
self.fh.write(self._HTML_HEADER)
def write(self, *args, **kwargs):
diff --git a/dshell/output/jsonout.py b/dshell/output/jsonout.py
index 8955031..f1e6121 100644
--- a/dshell/output/jsonout.py
+++ b/dshell/output/jsonout.py
@@ -25,7 +25,7 @@ def write(self, *args, **kwargs):
# before printing output
self.extra = False
if args and 'data' not in kwargs:
- kwargs['data'] = self.delim.join(map(str, args))
+ kwargs['data'] = self.delimiter.join(map(str, args))
jsondata = json.dumps(kwargs, ensure_ascii=self.ensure_ascii, default=self.json_default)
super().write(jsondata=jsondata)
diff --git a/dshell/output/netflowout.py b/dshell/output/netflowout.py
index 48469ca..f27cc3f 100644
--- a/dshell/output/netflowout.py
+++ b/dshell/output/netflowout.py
@@ -13,40 +13,58 @@ class NetflowOutput(Output):
separated by a forward-slash
For example:
--output=netflowout --oarg="group=clientip/serverip"
+ Note: Output when grouping is only generated at the end of analysis
+
+ A header row can be printed before output using --oarg header
"""
_DESCRIPTION = "Flow (connection overview) format output"
# Define two types of formats:
# Those for plugins handling individual packets (not really helpful)
- _PACKET_FORMAT = "%(ts)s %(sip)16s -> %(dip)16s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(msg)s\n"
- _PACKET6_FORMAT = "%(ts)s %(sip)40s -> %(dip)40s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(msg)s\n"
+ _PACKET_FORMAT = "%(ts)s %(sip)16s -> %(dip)16s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(data)s\n"
+ _PACKET6_FORMAT = "%(ts)s %(sip)40s -> %(dip)40s (%(sipcc)s -> %(dipcc)s) %(protocol)5s %(sport)6s %(dport)6s %(bytes)7s %(data)s\n"
+ _PACKET_PRETTY_HEADER = "[start timestamp] [source IP] -> [destination IP] ([source country] -> [destination country]) [protocol] [source port] [destination port] [bytes] [message data]\n"
# And those plugins handling full connections (more useful and common)
_CONNECTION_FORMAT = "%(starttime)s %(clientip)16s -> %(serverip)16s (%(clientcc)s -> %(servercc)s) %(protocol)5s %(clientport)6s %(serverport)6s %(clientpackets)5s %(serverpackets)5s %(clientbytes)7s %(serverbytes)7s %(duration)-.4fs %(data)s\n"
_CONNECTION6_FORMAT = "%(starttime)s %(clientip)40s -> %(serverip)40s (%(clientcc)s -> %(servercc)s) %(protocol)5s %(clientport)6s %(serverport)6s %(clientpackets)5s %(serverpackets)5s %(clientbytes)7s %(serverbytes)7s %(duration)-.4fs %(data)s\n"
+ _CONNECTION_PRETTY_HEADER = "[start timestamp] [client IP] -> [server IP] ([client country] -> [server country]) [protocol] [client port] [server port] [client packets] [server packets] [client bytes] [server bytes] [duration] [message data]\n"
# TODO decide if IPv6 formats are necessary, and how to switch between them
# and IPv4 formats
# Default to packets since those fields are in both types of object
_DEFAULT_FORMAT = _PACKET_FORMAT
def __init__(self, *args, **kwargs):
+ self.group = False
+ self.group_cache = {} # results will be stored here, if grouping
+ self.format_is_set = False
+ self.use_header = False
+ Output.__init__(self, *args, **kwargs)
+
+ def set_format(self, fmt, pretty_header=_PACKET_PRETTY_HEADER):
+ if self.use_header:
+ self.fh.write(str(pretty_header))
+ return super().set_format(fmt)
+
+ def set_oargs(self, **kwargs):
+ # Are we printing the format string as a file header?
+ self.use_header = kwargs.pop("header", False)
# Are we grouping the results, and by what fields?
if 'group' in kwargs:
self.group = True
- self.group_fields = kwargs['group'].split('/')
+ groupfields = kwargs.pop('group', '')
+ self.group_fields = groupfields.split('/')
else:
self.group = False
- self.group_cache = {} # results will be stored here, if grouping
- self.format_is_set = False
- Output.__init__(self, *args, **kwargs)
+ super().set_oargs(**kwargs)
def write(self, *args, **kwargs):
# Change output format depending on if we're handling a connection or
# a single packet
if not self.format_is_set:
if "clientip" in kwargs:
- self.set_format(self._CONNECTION_FORMAT)
+ self.set_format(self._CONNECTION_FORMAT, self._CONNECTION_PRETTY_HEADER)
else:
- self.set_format(self._PACKET_FORMAT)
+ self.set_format(self._PACKET_FORMAT, self._PACKET_PRETTY_HEADER)
self.format_is_set = True
if self.group:
@@ -55,7 +73,6 @@ def write(self, *args, **kwargs):
try:
key = tuple([kwargs[g] for g in self.group_fields])
except KeyError as e:
- self.logger.error("Could not group by key %s" % str(e))
Output.write(self, *args, **kwargs)
return
if key not in self.group_cache:
@@ -72,7 +89,7 @@ def write(self, *args, **kwargs):
def close(self):
if self.group:
self.group = False # we're done grouping, so turn it off
- for key in sorted(self.group_cache.keys()):
+ for key in self.group_cache.keys():
# write header by mapping key index with user's group list
self.fh.write(' '.join([
'%s=%s' % (self.group_fields[i], key[i]) for i in range(len(self.group_fields))])
@@ -83,4 +100,4 @@ def close(self):
self.fh.write("\n")
Output.close(self)
-obj = NetflowOutput
+obj = NetflowOutput
\ No newline at end of file
diff --git a/dshell/output/output.py b/dshell/output/output.py
index cf4556d..60b53e8 100644
--- a/dshell/output/output.py
+++ b/dshell/output/output.py
@@ -29,6 +29,9 @@ class Output:
fh : existing open file handle
file : filename to write to, assuming fh is not defined
mode : mode to open file, assuming fh is not defined (default 'w')
+ cbf : activate color blind friendly mode, colorout and htmlout output
+ modules use yellow/gold in place of red and different shades of
+ green/yellow/blue are used to help better differentiate between them
"""
_DEFAULT_FORMAT = "%(data)s\n"
_DEFAULT_TIME_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -37,7 +40,7 @@ class Output:
def __init__(
self, file=None, fh=None, mode='w', format=None, timeformat=None, delimiter=None, nobuffer=False,
- noclobber=False, extra=None, **unused_kwargs
+ noclobber=False, extra=None, cbf=False, **unused_kwargs
):
self.format_fields = []
self.timeformat = timeformat or self._DEFAULT_TIME_FORMAT
@@ -46,6 +49,7 @@ def __init__(
self.noclobber = noclobber
self.extra = extra
self.mode = mode
+ self.cbf = cbf
# Must define attributes even if they are setup in different function.
self.format_fields = None
@@ -82,16 +86,21 @@ def reset_fh(self, filename=None, fh=None, mode=None):
else:
self.fh = open(filename, self.mode)
- def set_oargs(self, format=None, noclobber=None, delimiter=None, timeformat=None, **unused_kwargs):
+ def set_oargs(self, format=None, noclobber=None, delimiter=None, timeformat=None, hex=None, **unused_kwargs):
"""
Process the standard oargs from the command line.
"""
if delimiter:
- self.delimiter = delimiter
+ if delimiter == "tab":
+ self.delimiter = '\t'
+ else:
+ self.delimiter = delimiter
if timeformat:
self.timeformat = timeformat
if noclobber:
self.noclobber = noclobber
+ if hex:
+ self.hexmode = hex
if format:
self.set_format(format)
diff --git a/dshell/plugins/flows/netflow.py b/dshell/plugins/flows/netflow.py
index 89a30e9..df9a7a8 100644
--- a/dshell/plugins/flows/netflow.py
+++ b/dshell/plugins/flows/netflow.py
@@ -9,10 +9,33 @@ class DshellPlugin(dshell.core.ConnectionPlugin):
def __init__(self, *args, **kwargs):
super().__init__(
name="Netflow",
- description="Collects and displays statistics about connections",
+ description="Collects and displays flow statistics about connections",
author="dev195",
bpf="ip or ip6",
output=NetflowOutput(label=__name__),
+ longdescription="""
+Collect and display flow statistics about connections.
+
+It will reassemble connections and print one row for each flow keyed by
+address four-tuple. Each row, by default, will have the following fields:
+
+- Start Time : the timestamp of the first packet for a connection
+- Client IP : the IP address of the host that initiated the connection
+- Server IP : the IP address of the host that receives the connection
+ (note: client/server designation is based on first packet seen for a connection)
+- Client Country : the country code for the client IP address
+- Server Country : the country code for the server IP address
+- Protocol : the layer-3 protocol of the connection
+- Client Port: port number used by client
+- Server Port: port number used by server
+- Client Packets : number of data-carrying packets from the client
+- Server Packets : number of data-carrying packets from the server
+ (note: packet counts ignore packets without data, e.g. handshakes, ACKs, etc.)
+- Client Bytes : total bytes sent by the client
+- Server Bytes : total bytes sent by the server
+- Duration : time between the first packet and final packet of a connection
+- Message Data: extra field not used by this plugin
+"""
)
def connection_handler(self, conn):
diff --git a/dshell/plugins/http/httpdump.py b/dshell/plugins/http/httpdump.py
index fcf200a..71362a5 100644
--- a/dshell/plugins/http/httpdump.py
+++ b/dshell/plugins/http/httpdump.py
@@ -163,6 +163,15 @@ def http_handler(self, conn, request, response):
kwargs['endtime'] = None
kwargs['serverbytes'] = 0
+ if post_params:
+ kwargs['post_params'] = post_params
+ if url_params:
+ kwargs['url_params'] = url_params
+ if client_cookie:
+ kwargs['client_cookie'] = client_cookie
+ if server_cookie:
+ kwargs['server_cookie'] = server_cookie
+
self.write('\n'.join(msg), **kwargs)
return conn, request, response
diff --git a/dshell/plugins/misc/followstream.py b/dshell/plugins/misc/followstream.py
index 5e73d57..10aa770 100644
--- a/dshell/plugins/misc/followstream.py
+++ b/dshell/plugins/misc/followstream.py
@@ -17,7 +17,7 @@ def __init__(self):
)
def connection_handler(self, conn):
- if (conn.clientbytes + conn.serverbytes > 0):
+ if conn.totalbytes > 0:
self.write(conn, **conn.info())
return conn
diff --git a/setup.py b/setup.py
index 2b896cb..9465134 100644
--- a/setup.py
+++ b/setup.py
@@ -2,11 +2,11 @@
setup(
name="Dshell",
- version="3.2.1",
+ version="3.2.3",
author="USArmyResearchLab",
description="An extensible network forensic analysis framework",
url="https://github.com/USArmyResearchLab/Dshell",
- python_requires='>=3.6',
+ python_requires='>=3.8',
packages=find_packages(),
package_data={
"dshell": ["data/dshellrc", "data/GeoIP/readme.txt"],
@@ -20,7 +20,7 @@
],
install_requires=[
"geoip2",
- "pcapy",
+ "pcapy-ng",
"pypacker",
"pyopenssl",
"elasticsearch",