Deduplicate event query table when no change is detected.

.github/workflows/go.yml

@@ -3,8 +3,8 @@ name: Linux Build All Arches
 on:
   pull_request:
     types: [closed]
-    branches:
-      - master
+#    branches:
+#      - master
 
 jobs:
   build:

actions/events.go

@@ -50,12 +50,44 @@ type EventTable struct {
 
 	config_obj *config_proto.Config
 
-	// This will be closed to signal we need to abort the current
-	// event queries.
+	// This will be closed to signal that we need to abort the
+	// current event queries.
 	Done chan bool
 	wg   sync.WaitGroup
 }
 
+func (self *EventTable) equal(events []*actions_proto.VQLCollectorArgs) bool {
+	if len(events) != len(self.Events) {
+		return false
+	}
+
+	for i := range self.Events {
+		lhs := self.Events[i]
+		rhs := events[i]
+
+		if len(lhs.Query) != len(rhs.Query) {
+			return false
+		}
+
+		for j := range lhs.Query {
+			if !proto.Equal(lhs.Query[j], rhs.Query[j]) {
+				return false
+			}
+		}
+
+		if len(lhs.Env) != len(rhs.Env) {
+			return false
+		}
+
+		for j := range lhs.Env {
+			if !proto.Equal(lhs.Env[j], rhs.Env[j]) {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 func (self *EventTable) Close() {
 	logger := logging.GetLogger(self.config_obj, &logging.ClientComponent)
 	logger.Info("Closing EventTable\n")
@@ -74,14 +106,28 @@ func GlobalEventTableVersion() uint64 {
 func update(
 	config_obj *config_proto.Config,
 	responder *responder.Responder,
-	table *actions_proto.VQLEventTable) (*EventTable, error) {
+	table *actions_proto.VQLEventTable) (*EventTable, error, bool) {
 
 	mu.Lock()
 	defer mu.Unlock()
 
 	// Only update the event table if we need to.
 	if table.Version <= GlobalEventTable.version {
-		return GlobalEventTable, nil
+		return GlobalEventTable, nil, false
+	}
+
+	// If the new update is identical to the old queries we wont
+	// restart. This can happen e.g. if the server changes label
+	// groups and recaculates the table version but the actual
+	// queries dont end up changing.
+	if GlobalEventTable.equal(table.Event) {
+		logger := logging.GetLogger(config_obj, &logging.ClientComponent)
+		logger.Info("Client event query update %v did not "+
+			"change queries, skipping", table.Version)
+
+		// Update the version only but keep queries the same.
+		GlobalEventTable.version = table.Version
+		return GlobalEventTable, nil, false
 	}
 
 	// Close the old table.
@@ -90,10 +136,9 @@ func update(
 	}
 
 	// Make a new table.
-	GlobalEventTable = NewEventTable(
-		config_obj, responder, table)
+	GlobalEventTable = NewEventTable(config_obj, responder, table)
 
-	return GlobalEventTable, nil
+	return GlobalEventTable, nil, true /* changed */
 }
 
 func NewEventTable(
@@ -119,9 +164,24 @@ func (self UpdateEventTable) Run(
 	arg *actions_proto.VQLEventTable) {
 
 	// Make a new table.
-	table, err := update(config_obj, responder, arg)
+	table, err, changed := update(config_obj, responder, arg)
 	if err != nil {
-		responder.Log(ctx, "Error updating global event table: %v", err)
+		responder.RaiseError(ctx, fmt.Sprintf(
+			"Error updating global event table: %v", err))
+		return
+	}
+
+	// No change required, skip it.
+	if !changed {
+		// We still need to write the new version
+		err = update_writeback(config_obj, arg)
+		if err != nil {
+			responder.RaiseError(ctx, fmt.Sprintf(
+				"Unable to write events to writeback: %v", err))
+		} else {
+			responder.Return(ctx)
+		}
+		return
 	}
 
 	logger := logging.GetLogger(config_obj, &logging.ClientComponent)
@@ -178,15 +238,24 @@ func (self UpdateEventTable) Run(
 		}(event)
 	}
 
-	// Store the event table in the Writeback file.
-	config_copy := proto.Clone(config_obj).(*config_proto.Config)
-	event_copy := proto.Clone(arg).(*actions_proto.VQLEventTable)
-	config_copy.Writeback.EventQueries = event_copy
-	err = config.UpdateWriteback(config_copy)
+	err = update_writeback(config_obj, arg)
 	if err != nil {
 		responder.RaiseError(ctx, fmt.Sprintf(
 			"Unable to write events to writeback: %v", err))
+		return
 	}
 
 	responder.Return(ctx)
 }
+
+func update_writeback(
+	config_obj *config_proto.Config,
+	event_table *actions_proto.VQLEventTable) error {
+
+	// Store the event table in the Writeback file.
+	config_copy := proto.Clone(config_obj).(*config_proto.Config)
+	event_copy := proto.Clone(event_table).(*actions_proto.VQLEventTable)
+	config_copy.Writeback.EventQueries = event_copy
+
+	return config.UpdateWriteback(config_copy)
+}

artifacts/definitions/Windows/EventLogs/Kerbroasting.yaml

@@ -14,42 +14,31 @@ description: |
   Kerbroasting can be used for privilege escalation or persistence by adding a
   SPN attribute to an unexpected account.
 
-  Log Source: Windows Security Event Log (Domain Controllers).  
-  Event ID: 4769 
-  Status: 0x0 (Audit Success)  
-  Ticket Encryption: 0x17 (RC4)  
-  Service Name: NOT krbtgt or NOT a system account (account name ends in $)  
-  TargetUserName: NOT a system account (*$@*)  
+  Log Source: Windows Security Event Log (Domain Controllers).
+  Event ID: 4769
+  Status: 0x0 (Audit Success)
+  Ticket Encryption: 0x17 (RC4)
+  Service Name: NOT krbtgt or NOT a system account (account name ends in $)
+  TargetUserName: NOT a system account (*$@*)
 
   Monitor and alert on unusual events with these conditions from an unexpected
-  IP.  
+  IP.
   Note: There are potential false positives so whitelist normal source IPs and
-  manage risk of insecure ticket generation.  
+  manage risk of insecure ticket generation.
 
 reference:
   - https://attack.mitre.org/techniques/T1208/
   - https://www.trustedsec.com/blog/art_of_kerberoast/
-  
+
 parameters:
   - name: EvtxGlob
     default: '%SystemRoot%\System32\winevt\logs\Security.evtx'
   - name: SearchVSS
     description: "Add VSS into query."
     type: bool
-    
+
 sources:
   - query: |
-      -- firstly set timebounds for performance
-      LET DateAfterTime <= if(condition=DateAfter,
-        then=timestamp(epoch=DateAfter), else=timestamp(epoch="1600-01-01"))
-      LET DateBeforeTime <= if(condition=DateBefore,
-        then=timestamp(epoch=DateBefore), else=timestamp(epoch="2200-01-01"))
-
-      -- Parse Log level dropdown selection
-      LET LogLevelRegex <= SELECT format(format="%v", args=Regex) as value
-        FROM parse_csv(filename=LogLevelMap, accessor="data")
-        WHERE Choice=LogLevel LIMIT 1
-
       -- expand provided glob into a list of paths on the file system (fs)
       LET fspaths <= SELECT FullPath
         FROM glob(globs=expand(path=EvtxGlob))
@@ -78,14 +67,14 @@ sources:
                     EventData.IpPort as IpPort,
                     FullPath
                 FROM parse_evtx(filename=FullPath)
-                WHERE 
+                WHERE
                     System.EventID.Value = 4769
                     AND EventData.TicketEncryptionType = 23
                     AND EventData.Status = 0
                     AND NOT EventData.ServiceName =~ "krbtgt|\\$$"
                     AND NOT EventData.TargetUserName =~ "\\$@"
           })
-    
+
 
       -- include VSS in calculation and deduplicate with GROUP BY by file
       LET include_vss = SELECT * FROM foreach(row=fspaths,
@@ -104,4 +93,4 @@ sources:
       -- return rows
       SELECT * FROM if(condition=SearchVSS,
         then=include_vss,
-        else=exclude_vss)
+        else=exclude_vss)

bin/golden.go

@@ -39,6 +39,7 @@ import (
 	logging "www.velocidex.com/golang/velociraptor/logging"
 	"www.velocidex.com/golang/velociraptor/reporting"
 	"www.velocidex.com/golang/velociraptor/services"
+	"www.velocidex.com/golang/velociraptor/services/hunt_dispatcher"
 	"www.velocidex.com/golang/velociraptor/startup"
 	vql_subsystem "www.velocidex.com/golang/velociraptor/vql"
 	"www.velocidex.com/golang/velociraptor/vql/tools"
@@ -147,6 +148,9 @@ func runTest(fixture *testFixture,
 	}
 	defer sm.Close()
 
+	err = sm.Start(hunt_dispatcher.StartHuntDispatcher)
+	kingpin.FatalIfError(err, "Starting services")
+
 	_, err = getRepository(config_obj)
 	kingpin.FatalIfError(err, "Loading extra artifacts")
 

logging/logging.go

@@ -365,7 +365,7 @@ func (self inMemoryLogWriter) Write(p []byte) (n int, err error) {
 
 	// Truncate too many logs
 	if len(memory_logs) > 1000 {
-		prelogs = nil
+		memory_logs = nil
 	}
 
 	memory_logs = append(memory_logs, string(p))

services/client_monitoring/client_monitoring.go

@@ -3,6 +3,11 @@
 // the GUI at any time.
 //
 // This service maintains access to the global event table.
+
+// NOTE: The client's event table will be updated when the client's
+// table's version is after:
+// 1. The global event table state was modified.
+// 2. Any label was updated for that client.
 package client_monitoring
 
 import (
@@ -80,15 +85,16 @@ func (self *ClientEventTable) CheckClientEventsVersion(
 	version := self.state.Version
 	self.mu.Unlock()
 
+	if client_version < version {
+		return true
+	}
+
+	// Now check the label group
 	labeler := services.GetLabeler()
 	if labeler == nil {
 		return false
 	}
 
-	if client_version < version {
-		return true
-	}
-
 	// If the client's labels have changed after their table
 	// timestamp, then they will need to update as well.
 	if client_version < labeler.LastLabelTimestamp(config_obj, client_id) {
@@ -286,22 +292,25 @@ func (self *ClientEventTable) ProcessArtifactModificationEvent(
 
 	// Determine if the modified artifact affects us.
 	is_relevant := func() bool {
-		// Ignore events that we sent.
+		// Ignore events that we sent ourselves.
 		if setter == self.id {
 			return false
 		}
 
+		// We could try to figure out if the artifact actually
+		// changed anythign but this is hard to know - not
+		// only do we need to look at the artifact in the
+		// event table but all dependencies as well. So for
+		// now we just recompile the event table when any
+		// artifact is changed.
 		return true
 	}
 
 	if is_relevant() {
-		// Recompile artifacts and update the version.
-		self.state.Version = uint64(self.clock.Now().UnixNano())
-
-		clear_caches(self.state)
-		err := self.compileState(ctx, config_obj, self.state)
+		err := self.load_from_file(ctx, config_obj)
 		if err != nil {
-			logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
+			logger := logging.GetLogger(
+				config_obj, &logging.FrontendComponent)
 			logger.Error("compileState: %v", err)
 		}
 	}
@@ -329,7 +338,13 @@ func (self *ClientEventTable) LoadFromFile(
 		return errors.New("Frontend not configured")
 	}
 
+	return self.load_from_file(ctx, config_obj)
+}
+
+func (self *ClientEventTable) load_from_file(
+	ctx context.Context, config_obj *config_proto.Config) error {
 	logger := logging.GetLogger(config_obj, &logging.FrontendComponent)
+	logger.Info("Reloading client monitoring tables from datastore\n")
 	db, err := datastore.GetDB(config_obj)
 	if err != nil {
 		return err

startup/startup.go

@@ -109,14 +109,6 @@ func StartupEssentialServices(sm *services.Service) error {
 		}
 	}
 
-	if services.GetHuntDispatcher() == nil {
-		// Hunt dispatcher manages client's hunt membership.
-		err := sm.Start(hunt_dispatcher.StartHuntDispatcher)
-		if err != nil {
-			return err
-		}
-	}
-
 	return nil
 }
 
@@ -149,6 +141,14 @@ func StartupFrontendServices(sm *services.Service) error {
 		}
 	}
 
+	if spec.HuntDispatcher {
+		// Hunt dispatcher manages client's hunt membership.
+		err := sm.Start(hunt_dispatcher.StartHuntDispatcher)
+		if err != nil {
+			return err
+		}
+	}
+
 	if spec.HuntManager {
 		err := sm.Start(hunt_manager.StartHuntManager)
 		if err != nil {