Recreate a container if inexplicably deleted

in CI, containers sometimes get purged

dgraphtest/dgraph.go

@@ -84,6 +84,7 @@ type dnode interface {
 	alphaURL(*LocalCluster) (string, error)
 	zeroURL(*LocalCluster) (string, error)
 	changeStatus(bool)
+	setContainerID(string)
 }
 
 type zero struct {
@@ -170,6 +171,10 @@ func (z *zero) changeStatus(isRunning bool) {
 	z.isRunning = isRunning
 }
 
+func (z *zero) setContainerID(cid string) {
+	z.containerID = cid
+}
+
 func (z *zero) assignURL(c *LocalCluster) (string, error) {
 	publicPort, err := publicPort(c.dcli, z, zeroHttpPort)
 	if err != nil {
@@ -364,6 +369,10 @@ func (a *alpha) changeStatus(isRunning bool) {
 	a.isRunning = isRunning
 }
 
+func (a *alpha) setContainerID(cid string) {
+	a.containerID = cid
+}
+
 func (a *alpha) zeroURL(c *LocalCluster) (string, error) {
 	return "", errNotImplemented
 }

dgraphtest/local_cluster.go

@@ -493,6 +493,22 @@ func (c *LocalCluster) StartAlpha(id int) error {
 func (c *LocalCluster) startContainer(dc dnode) error {
 	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
 	defer cancel()
+
+	// verify the container still exists
+	_, err := c.dcli.ContainerInspect(ctx, dc.cid())
+	if err != nil {
+		log.Printf("[WARNING] container %s (ID: %s) not found, attempting to recreate", dc.cname(), dc.cid())
+		newCID, createErr := c.createContainer(dc)
+		if createErr != nil {
+			return errors.Wrapf(createErr, "error recreating missing container [%v]", dc.cname())
+		}
+		switch node := dc.(type) {
+		case *alpha, *zero:
+			node.setContainerID(newCID)
+		}
+		log.Printf("[INFO] successfully recreated container %s with new ID: %s", dc.cname(), newCID)
+	}
+
 	if err := c.dcli.ContainerStart(ctx, dc.cid(), container.StartOptions{}); err != nil {
 		return errors.Wrapf(err, "error starting container [%v]", dc.cname())
 	}