batch the deletes

tburch · tburch · commit 317f7a458c98 · 2017-08-16T16:32:02.000-06:00
diff --git a/src/main/java/com/lowtuna/jsonblob/core/DataDirectoryCleanupJob.java b/src/main/java/com/lowtuna/jsonblob/core/DataDirectoryCleanupJob.java
@@ -70,8 +70,9 @@ public DateTime apply(@Nullable String input) {
           return;
         }
 
-        log.debug("Submitting BulkBlobDeleteJob for {} blobs in {}", toRemove.size(), dir);
-        executorService.submit(new BulkBlobDeleteJob(toRemove.keySet(), fileSystemJsonBlobManager, deleteEnabled));
+        log.debug("Submitting BulkBlobDeleteJobs for {} blobs in {}", toRemove.size(), dir);
+        List<List<String>> subSets = Lists.partition(Lists.newArrayList(toRemove.keySet()), 100);
+        subSets.parallelStream().forEach(list -> executorService.submit(new BulkBlobDeleteJob(Sets.newHashSet(list), fileSystemJsonBlobManager, deleteEnabled)));
       } catch (IOException e) {
         log.warn("Couldn't load metadata file from {}", dir.getAbsolutePath(), e);
       }

Original file line number	Diff line number	Diff line change
`@@ -70,8 +70,9 @@ public DateTime apply(@Nullable String input) {`
`70`	`70`	`return;`
`71`	`71`	`}`
`72`	`72`
`73`		`- log.debug("Submitting BulkBlobDeleteJob for {} blobs in {}", toRemove.size(), dir);`
`74`		`- executorService.submit(new BulkBlobDeleteJob(toRemove.keySet(), fileSystemJsonBlobManager, deleteEnabled));`
	`73`	`+ log.debug("Submitting BulkBlobDeleteJobs for {} blobs in {}", toRemove.size(), dir);`
	`74`	`+ List<List<String>> subSets = Lists.partition(Lists.newArrayList(toRemove.keySet()), 100);`
	`75`	`+ subSets.parallelStream().forEach(list -> executorService.submit(new BulkBlobDeleteJob(Sets.newHashSet(list), fileSystemJsonBlobManager, deleteEnabled)));`
`75`	`76`	`} catch (IOException e) {`
`76`	`77`	`log.warn("Couldn't load metadata file from {}", dir.getAbsolutePath(), e);`
`77`	`78`	`}`