Consolidate logic for running randomized tests in scripts

script/randomized-test-ci

@@ -1,50 +1,63 @@
-#!/bin/bash
+#!/usr/bin/env node --redirect-warnings=/dev/null
 
-set -u
+const fs = require('fs')
+const {randomBytes} = require('crypto')
+const {execFileSync} = require('child_process')
+const {minimizeTestPlan, buildTests, runTests} = require('./randomized-test-minimize');
 
-: $ZED_SERVER_URL
+const {ZED_SERVER_URL, ZED_CLIENT_SECRET_TOKEN} = process.env
-: $ZED_CLIENT_SECRET_TOKEN
+if (!ZED_SERVER_URL) throw new Error('Missing env var `ZED_SERVER_URL`')
+if (!ZED_CLIENT_SECRET_TOKEN) throw new Error('Missing env var `ZED_CLIENT_SECRET_TOKEN`')
 
-# Compile the tests first
+main()
-mkdir -p target
-cargo test --release --lib --package collab --no-run
-if [[ $? != 0 ]]; then
-  echo "Build failed"
-  exit 1
-fi
 
-LOG_FILE=target/randomized-tests.log
+async function main() {
-MIN_PLAN=target/test-plan.min.json
+  buildTests()
-export SAVE_PLAN=target/test-plan.json
-export OPERATIONS=200
-export ITERATIONS=100000
-export SEED=$(od -A n -N 8 -t u8 /dev/urandom | xargs)
 
-echo "Starting seed: ${SEED}"
+  const seed = randomU64();
+  const commit = execFileSync(
+    'git',
+    ['rev-parse', 'HEAD'],
+    {encoding: 'utf8'}
+  ).trim()
 
-cargo test --release --lib --package collab random 2>&1 > $LOG_FILE
+  console.log("commit:", commit)
-if [[ $? == 0 ]]; then
+  console.log("starting seed:", seed)
-  echo "Tests passed"
-  exit 0
-fi
 
-failing_seed=$(script/randomized-test-minimize $SAVE_PLAN $MIN_PLAN)
+  const planPath = 'target/test-plan.json'
+  const minPlanPath = 'target/test-plan.min.json'
+  const failingSeed = runTests({
+    SEED: seed,
+    SAVE_PLAN: planPath,
+    ITERATIONS: 50000,
+    OPERATIONS: 200,
+  })
 
-# If the tests failed, find the failing seed in the logs
+  if (!failingSeed) {
-commit=$(git rev-parse HEAD)
+    console.log("tests passed")
-failing_plan=$(cat $MIN_PLAN)
+    return
-request="{
+  }
-  \"seed\": \"${failing_seed}\",
-  \"commit\": \"${commit}\",
-  \"token\": \"${ZED_CLIENT_SECRET_TOKEN}\",
-  \"plan\": ${failing_plan}
-}"
 
-echo "Reporting test failure."
+  console.log("found failure at seed", failingSeed)
-echo $request
+  const minimizedSeed = minimizeTestPlan(planPath, minPlanPath)
+  const minimizedPlan = JSON.parse(fs.readFileSync(minPlanPath, 'utf8'))
 
-curl \
+  const url = `${ZED_SERVER_URL}/api/randomized_test_failure`
-    -X POST \
+  const body = {
-    -H "Content-Type: application/json" \
+    seed: minimizedSeed,
-    -d "${request}" \
+    token: ZED_CLIENT_SECRET_TOKEN,
-    "${ZED_SERVER_URL}/api/randomized_test_failure"
+    plan: minimizedPlan,
+    commit: commit,
+  }
+  await fetch(url, {
+    method: 'POST',
+    headers: {"Content-Type": "application/json"},
+    body: JSON.stringify(body)
+  })
+}
+
+function randomU64() {
+  const bytes = randomBytes(8)
+  const hexString = bytes.reduce(((string, byte) => string + byte.toString(16)), '')
+  return BigInt('0x' + hexString).toString(10)
+}

script/randomized-test-minimize

@@ -4,85 +4,109 @@ const fs = require('fs')
 const path = require('path')
 const {spawnSync} = require('child_process')
 
-if (process.argv.length < 4) {
-  process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
-  process.exit(1)
-}
-
-const inputPlanPath = process.argv[2]
-const outputPlanPath = process.argv[3]
-const startIndex = parseInt(process.argv[4]) || 0
-
-const tempPlanPath = inputPlanPath + '.try'
-
 const FAILING_SEED_REGEX = /failing seed: (\d+)/ig
+const CARGO_TEST_ARGS = [
+  '--release',
+  '--lib',
+  '--package', 'collab',
+  'random_collaboration',
+]
 
-fs.copyFileSync(inputPlanPath, outputPlanPath)
+if (require.main === module) {
-let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+  if (process.argv.length < 4) {
-
+    process.stderr.write("usage: script/randomized-test-minimize <input-plan> <output-plan> [start-index]\n")
-process.stderr.write("minimizing failing test plan...\n")
+    process.exit(1)
-for (let ix = startIndex; ix < testPlan.length; ix++) {
-  // Skip 'MutateClients' entries, since they themselves are not single operations.
-  if (testPlan[ix].MutateClients) {
-    continue
   }
 
-  // Remove a row from the test plan
+  minimizeTestPlan(
-  const newTestPlan = testPlan.slice()
+    process.argv[2],
-  newTestPlan.splice(ix, 1)
+    process.argv[3],
-  fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+    parseInt(process.argv[4]) || 0
+  );
+}
 
-  process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+function minimizeTestPlan(
+  inputPlanPath,
+  outputPlanPath,
+  startIndex = 0
+) {
+  const tempPlanPath = inputPlanPath + '.try'
 
-  const failingSeed = runTestsForPlan(tempPlanPath, 500)
+  fs.copyFileSync(inputPlanPath, outputPlanPath)
+  let testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
 
-  // If the test failed, keep the test plan with the removed row. Reload the test
+  process.stderr.write("minimizing failing test plan...\n")
-  // plan from the JSON file, since the test itself will remove any operations
+  for (let ix = startIndex; ix < testPlan.length; ix++) {
-  // which are no longer valid before saving the test plan.
+    // Skip 'MutateClients' entries, since they themselves are not single operations.
-  if (failingSeed != null) {
+    if (testPlan[ix].MutateClients) {
-    process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+      continue
-    fs.copyFileSync(tempPlanPath, outputPlanPath)
+    }
-    testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+
-    ix--
+    // Remove a row from the test plan
-  } else {
+    const newTestPlan = testPlan.slice()
-    process.stderr.write(` - keep.\n`)
+    newTestPlan.splice(ix, 1)
+    fs.writeFileSync(tempPlanPath, serializeTestPlan(newTestPlan), 'utf8');
+
+    process.stderr.write(`${ix}/${testPlan.length}: ${JSON.stringify(testPlan[ix])}`)
+    const failingSeed = runTests({
+      SEED: '0',
+      LOAD_PLAN: tempPlanPath,
+      SAVE_PLAN: tempPlanPath,
+      ITERATIONS: '500'
+    })
+
+    // If the test failed, keep the test plan with the removed row. Reload the test
+    // plan from the JSON file, since the test itself will remove any operations
+    // which are no longer valid before saving the test plan.
+    if (failingSeed != null) {
+      process.stderr.write(` - remove. failing seed: ${failingSeed}.\n`)
+      fs.copyFileSync(tempPlanPath, outputPlanPath)
+      testPlan = JSON.parse(fs.readFileSync(outputPlanPath, 'utf8'))
+      ix--
+    } else {
+      process.stderr.write(` - keep.\n`)
+    }
+  }
+
+  fs.unlinkSync(tempPlanPath)
+
+  // Re-run the final minimized plan to get the correct failing seed.
+  // This is a workaround for the fact that the execution order can
+  // slightly change when replaying a test plan after it has been
+  // saved and loaded.
+  const failingSeed = runTests({
+    SEED: '0',
+    ITERATIONS: '5000',
+    LOAD_PLAN: outputPlanPath,
+  })
+
+  process.stderr.write(`final test plan: ${outputPlanPath}\n`)
+  process.stderr.write(`final seed: ${failingSeed}\n`)
+  return failingSeed
+}
+
+function buildTests() {
+  const {status} = spawnSync('cargo', ['test', '--no-run', ...CARGO_TEST_ARGS], {
+    stdio: 'inherit',
+    encoding: 'utf8',
+    env: {
+      ...process.env,
+    }
+  });
+  if (status !== 0) {
+    throw new Error('build failed')
   }
 }
 
-fs.unlinkSync(tempPlanPath)
+function runTests(env) {
-
+  const {status, stdout} = spawnSync('cargo', ['test', ...CARGO_TEST_ARGS], {
-// Re-run the final minimized plan to get the correct failing seed.
+    stdio: 'pipe',
-// This is a workaround for the fact that the execution order can
+    encoding: 'utf8',
-// slightly change when replaying a test plan after it has been
+    env: {
-// saved and loaded.
+      ...process.env,
-const failingSeed = runTestsForPlan(outputPlanPath, 5000)
+      ...env,
-
-process.stderr.write(`final test plan: ${outputPlanPath}\n`)
-process.stderr.write(`final seed: ${failingSeed}\n`)
-console.log(failingSeed)
-
-function runTestsForPlan(path, iterations) {
-  const {status, stdout, stderr} = spawnSync(
-    'cargo',
-    [
-      'test',
-      '--release',
-      '--lib',
-      '--package', 'collab',
-      'random_collaboration'
-    ],
-    {
-      stdio: 'pipe',
-      encoding: 'utf8',
-      env: {
-        ...process.env,
-        'SEED': 0,
-        'LOAD_PLAN': path,
-        'SAVE_PLAN': path,
-        'ITERATIONS': String(iterations),
-      }
     }
-  );
+  });
 
   if (status !== 0) {
     FAILING_SEED_REGEX.lastIndex = 0
@@ -102,3 +126,7 @@ function runTestsForPlan(path, iterations) {
 function serializeTestPlan(plan) {
   return "[\n" + plan.map(row => JSON.stringify(row)).join(",\n") + "\n]\n"
 }
+
+exports.buildTests = buildTests
+exports.runTests = runTests
+exports.minimizeTestPlan = minimizeTestPlan