randomized-test-minimize

#!/usr/bin/env node --redirect-warnings=/dev/null

const fs = require("fs");
const path = require("path");
const { spawnSync } = require("child_process");

const FAILING_SEED_REGEX = /failing seed: (\d+)/gi;
const CARGO_TEST_ARGS = ["--release", "--lib", "--package", "collab"];

if (require.main === module) {
  if (process.argv.length < 4) {
    process.stderr.write(
      "usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n",
    );
    process.exit(1);
  }

  minimizeTestPlan(
    process.argv[2],
    process.argv[3],
    parseInt(process.argv[4]) || 0,
  );
}

function minimizeTestPlan(inputPlanPath, outputPlanPath, startIndex = 0) {
  const tempPlanPath = inputPlanPath + ".try";

  fs.copyFileSync(inputPlanPath, outputPlanPath);
  let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));

  process.stderr.write("minimizing failing test plan...\n");
  for (let ix = startIndex; ix < testPlan.length; ix++) {
    // Skip 'MutateClients' entries, since they themselves are not single operations.
    if (testPlan[ix].MutateClients) {
      continue;
    }

    // Remove a row from the test plan
    const newTestPlan = testPlan.slice();
    newTestPlan.splice(ix, 1);
    fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), "utf8");

    process.stderr.write(
      `${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`,
    );
    const failingSeed = runTests({
      SEED: "0",
      LOAD_PLAN: tempPlanPath,
      SAVE_PLAN: tempPlanPath,
      ITERATIONS: "500",
    });

    // If the test failed, keep the test plan with the removed row. Reload the test
    // plan from the JSON file, since the test itself will remove any operations
    // which are no longer valid before saving the test plan.
    if (failingSeed != null) {
      process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`);
      fs.copyFileSync(tempPlanPath, outputPlanPath);
      testPlan = JSON.parse(fs.readFileSync(outputPlanPath, "utf8"));
      ix--;
    } else {
      process.stderr.write(` - keep.\n`);
    }
  }

  fs.unlinkSync(tempPlanPath);

  // Re-run the final minimized plan to get the correct failing seed.
  // This is a workaround for the fact that the execution order can
  // slightly change when replaying a test plan after it has been
  // saved and loaded.
  const failingSeed = runTests({
    SEED: "0",
    ITERATIONS: "5000",
    LOAD_PLAN: outputPlanPath,
  });

  process.stderr.write(`final test plan: ${outputPlanPath}\n`);
  process.stderr.write(`final seed: ${failingSeed}\n`);
  return failingSeed;
}

function buildTests() {
  const { status } = spawnSync(
    "cargo",
    ["test", "--no-run", ...CARGO_TEST_ARGS],
    {
      stdio: "inherit",
      encoding: "utf8",
      env: {
        ...process.env,
      },
    },
  );
  if (status !== 0) {
    throw new Error("build failed");
  }
}

function runTests(env) {
  const { status, stdout } = spawnSync(
    "cargo",
    ["test", ...CARGO_TEST_ARGS, "random_project_collaboration"],
    {
      stdio: "pipe",
      encoding: "utf8",
      env: {
        ...process.env,
        ...env,
      },
    },
  );

  if (status !== 0) {
    FAILING_SEED_REGEX.lastIndex = 0;
    const match = FAILING_SEED_REGEX.exec(stdout);
    if (!match) {
      process.stderr.write("test failed, but no failing seed found:\n");
      process.stderr.write(stdout);
      process.stderr.write("\n");
      process.exit(1);
    }
    return match[1];
  } else {
    return null;
  }
}

function serializeTestPlan(plan) {
  return "[\n" + plan.map((row) => JSON.stringify(row)).join(",\n") + "\n]\n";
}

exports.buildTests = buildTests;
exports.runTests = runTests;
exports.minimizeTestPlan = minimizeTestPlan;