diff --git a/filter_plugins/custom.py b/filter_plugins/custom.py index 22177cdd..8a66c8fc 100644 --- a/filter_plugins/custom.py +++ b/filter_plugins/custom.py @@ -23,7 +23,7 @@ def array_to_str(values=[],separator=','): def extract_role_users(users={},exclude_users=[]): role_users=[] - for user,details in users.iteritems(): + for user,details in users.items(): if user not in exclude_users and "roles" in details: for role in details["roles"]: role_users.append(role+":"+user) @@ -54,4 +54,4 @@ def filters(self): 'array_to_str':array_to_str, 'extract_role_users':extract_role_users, 'remove_reserved':remove_reserved, - 'filename':filename} \ No newline at end of file + 'filename':filename} diff --git a/templates/log4j2.properties.j2 b/templates/log4j2.properties.j2 index dbfb23e0..b1b288de 100644 --- a/templates/log4j2.properties.j2 +++ b/templates/log4j2.properties.j2 @@ -9,6 +9,180 @@ appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n +{% if (es_version is version_compare('7.0.0', '>=')) %} +######## Server JSON ############################ +appender.rolling.type = RollingFile +appender.rolling.name = rolling +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server + +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz +appender.rolling.policies.type = Policies +appender.rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling.policies.time.interval = 1 +appender.rolling.policies.time.modulate = true +appender.rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling.policies.size.size = 128MB +appender.rolling.strategy.type = DefaultRolloverStrategy +appender.rolling.strategy.fileIndex = nomax +appender.rolling.strategy.action.type = Delete +appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling.strategy.action.condition.type = IfFileName +appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ +rootLogger.level = info +rootLogger.appenderRef.console.ref = console +rootLogger.appenderRef.rolling.ref = rolling +rootLogger.appenderRef.rolling_old.ref = rolling_old + +######## Deprecation JSON ####################### +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json +appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation + +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 +################################################# +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old +logger.deprecation.additivity = false + +######## Search slowlog JSON #################### +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.json +appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog + +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%i.json.gz +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.size.size = 1GB +appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling.strategy.max = 4 +################################################# +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old +logger.index_search_slowlog_rolling.additivity = false + +######## Indexing slowlog JSON ################## +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.json +appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog + +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%i.json.gz +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling.strategy.max = 4 +################################################# +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old +logger.index_indexing_slowlog.additivity = false + + +appender.audit_rolling.type = RollingFile +appender.audit_rolling.name = audit_rolling +appender.audit_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit.json +appender.audit_rolling.layout.type = PatternLayout +appender.audit_rolling.layout.pattern = {{ '{' }}\ + "@timestamp":"%d{ISO8601}"\ + {{ '%varsNotEmpty{, "node.name":"%enc{%map{node.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "node.id":"%enc{%map{node.id}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "host.name":"%enc{%map{host.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "host.ip":"%enc{%map{host.ip}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "event.type":"%enc{%map{event.type}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "event.action":"%enc{%map{event.action}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.name":"%enc{%map{user.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.run_by.name":"%enc{%map{user.run_by.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.run_as.name":"%enc{%map{user.run_as.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.realm":"%enc{%map{user.realm}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.run_by.realm":"%enc{%map{user.run_by.realm}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.run_as.realm":"%enc{%map{user.run_as.realm}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "user.roles":%map{user.roles}}' }}\ + {{ '%varsNotEmpty{, "origin.type":"%enc{%map{origin.type}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "origin.address":"%enc{%map{origin.address}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "realm":"%enc{%map{realm}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "url.path":"%enc{%map{url.path}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "url.query":"%enc{%map{url.query}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "request.method":"%enc{%map{request.method}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "request.body":"%enc{%map{request.body}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "request.id":"%enc{%map{request.id}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "action":"%enc{%map{action}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "indices":%map{indices}}' }}\ + {{ '%varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}' }}\ + {{ '%varsNotEmpty{, "event.category":"%enc{%map{event.category}}{JSON}"}' }}\ + {{ '}' }}%n +# "node.name" node name from the `elasticsearch.yml` settings +# "node.id" node id which should not change between cluster restarts +# "host.name" unresolved hostname of the local node +# "host.ip" the local bound ip (i.e. the ip listening for connections) +# "event.type" a received REST request is translated into one or more transport requests. This indicates which processing layer generated the event "rest" or "transport" (internal) +# "event.action" the name of the audited event, eg. "authentication_failed", "access_granted", "run_as_granted", etc. +# "user.name" the subject name as authenticated by a realm +# "user.run_by.name" the original authenticated subject name that is impersonating another one. +# "user.run_as.name" if this "event.action" is of a run_as type, this is the subject name to be impersonated as. +# "user.realm" the name of the realm that authenticated "user.name" +# "user.run_by.realm" the realm name of the impersonating subject ("user.run_by.name") +# "user.run_as.realm" if this "event.action" is of a run_as type, this is the realm name the impersonated user is looked up from +# "user.roles" the roles array of the user; these are the roles that are granting privileges +# "origin.type" it is "rest" if the event is originating (is in relation to) a REST request; possible other values are "transport" and "ip_filter" +# "origin.address" the remote address and port of the first network hop, i.e. a REST proxy or another cluster node +# "realm" name of a realm that has generated an "authentication_failed" or an "authentication_successful"; the subject is not yet authenticated +# "url.path" the URI component between the port and the query string; it is percent (URL) encoded +# "url.query" the URI component after the path and before the fragment; it is percent (URL) encoded +# "request.method" the method of the HTTP request, i.e. one of GET, POST, PUT, DELETE, OPTIONS, HEAD, PATCH, TRACE, CONNECT +# "request.body" the content of the request body entity, JSON escaped +# "request.id" a synthentic identifier for the incoming request, this is unique per incoming request, and consistent across all audit events generated by that request +# "action" an action is the most granular operation that is authorized and this identifies it in a namespaced way (internal) +# "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal) +# "indices" the array of indices that the "action" is acting upon +# "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header +# "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array) +# "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event +# "rule" name of the applied rulee if the "origin.type" is "ip_filter" +# "event.category" fixed value "elasticsearch-audit" + +appender.audit_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_audit-%d{yyyy-MM-dd}.json +appender.audit_rolling.policies.type = Policies +appender.audit_rolling.policies.time.type = TimeBasedTriggeringPolicy +appender.audit_rolling.policies.time.interval = 1 +appender.audit_rolling.policies.time.modulate = true + +logger.xpack_security_audit_logfile.name = org.elasticsearch.xpack.security.audit.logfile.LoggingAuditTrail +logger.xpack_security_audit_logfile.level = info +logger.xpack_security_audit_logfile.appenderRef.audit_rolling.ref = audit_rolling +logger.xpack_security_audit_logfile.additivity = false + +logger.xmlsig.name = org.apache.xml.security.signature.XMLSignature +logger.xmlsig.level = error +logger.samlxml_decrypt.name = org.opensaml.xmlsec.encryption.support.Decrypter +logger.samlxml_decrypt.level = fatal +logger.saml2_decrypt.name = org.opensaml.saml.saml2.encryption.Decrypter +logger.saml2_decrypt.level = fatal +{% else %} appender.rolling.type = RollingFile appender.rolling.name = rolling {% if (es_version is version_compare('6.0.0', '<')) %} @@ -38,7 +212,9 @@ appender.rolling.strategy.action.condition.type = IfFileName appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB + {% endif %} + rootLogger.level = info rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling @@ -115,3 +291,4 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index logger.index_indexing_slowlog.level = trace logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling logger.index_indexing_slowlog.additivity = false +{% endif %}