Merge pull request Netflix#223 from ebukoski/master

Add a URL target to add events through HTTP GET; more Calendar logging

Author: ebukoski

src/main/java/com/netflix/simianarmy/basic/BasicCalendar.java

@@ -24,6 +24,8 @@
 import java.util.TreeSet;
 
 import org.apache.commons.lang.Validate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.netflix.simianarmy.Monkey;
 import com.netflix.simianarmy.MonkeyCalendar;
@@ -35,6 +37,9 @@
  */
 public class BasicCalendar implements MonkeyCalendar {
 
+    /** The Constant LOGGER. */
+    private static final Logger LOGGER = LoggerFactory.getLogger(BasicCalendar.class);
+
     /** The open hour. */
     private final int openHour;
 
@@ -78,6 +83,23 @@ public BasicCalendar(int open, int close, TimeZone timezone) {
         closeHour = close;
         tz = timezone;
     }
+    
+    /**
+     * Instantiates a new basic calendar.
+     *
+     * @param open
+     *            the open hour
+     * @param close
+     *            the close hour
+     * @param timezone
+     *            the timezone
+     */
+    public BasicCalendar(MonkeyConfiguration cfg, int open, int close, TimeZone timezone) {
+        this.cfg = cfg;
+        openHour = open;
+        closeHour = close;
+        tz = timezone;
+    }
 
     /** {@inheritDoc} */
     @Override
@@ -100,25 +122,32 @@ public Calendar now() {
     /** {@inheritDoc} */
     @Override
     public boolean isMonkeyTime(Monkey monkey) {
-        if (cfg != null && cfg.getStrOrElse("simianarmy.calendar.isMonkeyTime", null) != null) {
-            return cfg.getBool("simianarmy.calendar.isMonkeyTime");
+        if (cfg != null && cfg.getStr("simianarmy.calendar.isMonkeyTime") != null) {
+        	boolean monkeyTime = cfg.getBool("simianarmy.calendar.isMonkeyTime");
+        	String msg = monkeyTime ? "Time for monkey." : "Not time for monkey.";
+        	LOGGER.debug("isMonkeyTime: Found property 'simianarmy.calendar.isMonkeyTime': " + monkeyTime + ". " + msg);
+            return monkeyTime;
         }
 
         Calendar now = now();
         int dow = now.get(Calendar.DAY_OF_WEEK);
         if (dow == Calendar.SATURDAY || dow == Calendar.SUNDAY) {
+        	LOGGER.debug("isMonkeyTime: Happy Weekend! Not time for monkey.");
             return false;
         }
 
         int hour = now.get(Calendar.HOUR_OF_DAY);
         if (hour < openHour || hour > closeHour) {
+        	LOGGER.debug("isMonkeyTime: Not inside open hours. Not time for monkey.");
             return false;
         }
 
         if (isHoliday(now)) {
+        	LOGGER.debug("isMonkeyTime: Happy Holiday! Not time for monkey.");
             return false;
         }
 
+    	LOGGER.debug("isMonkeyTime: Time for monkey.");
         return true;
     }
 

src/main/java/com/netflix/simianarmy/basic/janitor/BasicJanitorMonkey.java

@@ -68,6 +68,12 @@ public class BasicJanitorMonkey extends JanitorMonkey {
     /** Keep track of the number of monkey runs */
     protected final AtomicLong monkeyRuns = new AtomicLong(0);    
 
+    /** Keep track of the number of monkey errors */
+    protected final AtomicLong monkeyErrors = new AtomicLong(0);    
+    
+    /** Emit a servor signal to track the running monkey */
+    protected final AtomicLong monkeyRunning = new AtomicLong(0);    
+    
     /**
      * Instantiates a new basic janitor monkey.
      *
@@ -101,9 +107,15 @@ public void doMonkeyBusiness() {
         } else {
             LOGGER.info(String.format("Marking resources with %d janitors.", janitors.size()));
             monkeyRuns.incrementAndGet();
+            monkeyRunning.set(1);
             for (AbstractJanitor janitor : janitors) {
-                LOGGER.info(String.format("Running janitor for region %s", janitor.getRegion()));
-                janitor.markResources();
+                LOGGER.info(String.format("Running %s janitor for region %s", janitor.getResourceType(), janitor.getRegion()));
+                try {
+                	janitor.markResources();
+                } catch (Exception e) {
+                	monkeyErrors.incrementAndGet();
+                	LOGGER.error(String.format("Got an exception while %s janitor was marking for region %s", janitor.getResourceType()), janitor.getRegion(), e);
+                }
                 LOGGER.info(String.format("Marked %d resources of type %s in the last run.",
                         janitor.getMarkedResources().size(), janitor.getResourceType().name()));
                 LOGGER.info(String.format("Unmarked %d resources of type %s in the last run.",
@@ -118,7 +130,12 @@ public void doMonkeyBusiness() {
 
             LOGGER.info(String.format("Cleaning resources with %d janitors.", janitors.size()));
             for (AbstractJanitor janitor : janitors) {
-                janitor.cleanupResources();
+            	try {
+            		janitor.cleanupResources();
+                } catch (Exception e) {
+                	monkeyErrors.incrementAndGet();
+                	LOGGER.error(String.format("Got an exception while %s janitor was cleaning for region %s", janitor.getResourceType()), janitor.getRegion(), e);
+                }
                 LOGGER.info(String.format("Cleaned %d resources of type %s in the last run.",
                         janitor.getCleanedResources().size(), janitor.getResourceType()));
                 LOGGER.info(String.format("Failed to clean %d resources of type %s in the last run.",
@@ -127,6 +144,7 @@ public void doMonkeyBusiness() {
             if (cfg.getBoolOrElse(NS + "summaryEmail.enabled", true)) {
                 sendJanitorSummaryEmail();
             }
+        	monkeyRunning.set(0);
         }
     }
 
@@ -237,4 +255,15 @@ private boolean isJanitorMonkeyEnabled() {
     public long getMonkeyRuns() {
       return monkeyRuns.get();
     }
+
+    @Monitor(name="errors", type=DataSourceType.GAUGE)
+    public long getMonkeyErrors() {
+      return monkeyErrors.get();
+    }
+
+    @Monitor(name="running", type=DataSourceType.GAUGE)
+    public long getMonkeyRunning() {
+      return monkeyRunning.get();
+    }
+    
 }

src/main/java/com/netflix/simianarmy/janitor/AbstractJanitor.java

@@ -106,6 +106,9 @@ public String getRegion() {
     private boolean leashed;
 
     private final MonkeyRecorder recorder;
+       
+    /** The number of resources that have been checked on this run. */
+    private int checkedResourcesCount;
 
     /**
      * Sets the flag to indicate if the janitor is leashed.
@@ -217,13 +220,15 @@ public ResourceType getResourceType() {
     public void markResources() {
         markedResources.clear();        
         unmarkedResources.clear();
+        checkedResourcesCount = 0;
         Map<String, Resource> trackedMarkedResources = getTrackedMarkedResources();
 
         List<Resource> crawledResources = crawler.resources(resourceType);
         LOGGER.info(String.format("Looking for cleanup candidate in %d crawled resources.",
                 crawledResources.size()));
         Date now = calendar.now().getTime();
         for (Resource resource : crawledResources) {
+        	checkedResourcesCount++;
             Resource trackedResource = trackedMarkedResources.get(resource.getId());
             if (!ruleEngine.isValid(resource)) {
                 // If the resource is already marked, ignore it
@@ -238,6 +243,7 @@ public void markResources() {
                 if (!leashed) {
                     if (recorder != null) {
                         Event evt = recorder.newEvent(Type.JANITOR, EventTypes.MARK_RESOURCE, region, resource.getId());
+                        addFieldsAndTagsToEvent(resource, evt);
                         recorder.recordEvent(evt);
                     }
                     resourceTracker.addOrUpdate(resource);
@@ -257,6 +263,7 @@ public void markResources() {
                     if (recorder != null) {
                         Event evt = recorder.newEvent(
                                 Type.JANITOR, EventTypes.UNMARK_RESOURCE, region, resource.getId());
+                        addFieldsAndTagsToEvent(resource, evt);
                         recorder.recordEvent(evt);
                     }
                     resourceTracker.addOrUpdate(resource);
@@ -337,8 +344,11 @@ public void cleanupResources() {
 	 * @param event    the event that will hold the source data as additional fields
      */
     private void addFieldsAndTagsToEvent(Resource resource, Event event) {
-    	for(String key : resource.getAllTagKeys()) {
-    		event.addField(key, resource.getTag(key));
+    	if (resource == null) return;
+    	if (resource.getAllTagKeys() != null) {
+	    	for(String key : resource.getAllTagKeys()) {
+	    		event.addField(key, resource.getTag(key));
+	    	}
     	}
     	event.addField("ResourceDescription", resource.getDescription());
     	event.addField("ResourceType", resource.getResourceType().toString());		
@@ -448,4 +458,10 @@ public int getFailedToCleanResourcesCount() {
     public int getUnmarkedResourcesCount() {
       return unmarkedResources.size();
     }    
+    
+    @Monitor(name="checkedResourcesCount", type=DataSourceType.GAUGE)
+    public int getCheckedResourcesCount() {
+      return checkedResourcesCount;
+    }    
+    
 }

src/main/java/com/netflix/simianarmy/resources/janitor/JanitorMonkeyResource.java

@@ -24,6 +24,7 @@
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.Response;
 import javax.ws.rs.core.UriInfo;
@@ -72,6 +73,23 @@ public JanitorMonkeyResource(JanitorMonkey monkey) {
     public JanitorMonkeyResource() {
         this.monkey = MonkeyRunner.getInstance().factory(JanitorMonkey.class);
     }
+    
+    /**
+     * GET /api/v1/janitor/addEvent will try to a add a new event with the information in the url query string.
+     * This is the same as the regular POST addEvent except through a query string. This technically isn't
+     * very REST-ful as it is a GET call that creates an Opt-out/in event, but is a convenience method 
+     * for exposing opt-in/opt-out functionality more directly, for example in an email notification.
+     *
+     * @param eventType eventType from the query string
+     * @param resourceId resourceId from the query string
+     * @return the response
+     * @throws IOException
+     */
+    @GET @Path("addEvent")
+    public Response addEventThroughHttpGet( @QueryParam("eventType") String eventType,  @QueryParam("resourceId") String resourceId) throws IOException {
+    	String content = "{\"eventType\":\"" + eventType + "\",\"resourceId\":\"" + resourceId + "\"}";    	
+        return addEvent(content);        
+    }
 
     /**
      * POST /api/v1/janitor will try a add a new event with the information in the url context.