feat: proposal pre/post test steps

packages/synthetic-chain/package.json

@@ -25,13 +25,15 @@
     "chalk": "^5.4.1",
     "cosmjs-types": "^0.9.0",
     "execa": "^9.5.2",
-    "glob": "^11.0.0"
+    "glob": "^11.0.0",
+    "tmp": "0.2.3"
   },
   "devDependencies": {
     "@agoric/cosmic-proto": "0.5.0-u18.5",
     "@types/better-sqlite3": "^7.6.11",
     "@types/glob": "^8.1.0",
     "@types/node": "^18.19.50",
+    "@types/tmp": "0.2.6",
     "ava": "^6.2.0",
     "ts-blank-space": "^0.5.0",
     "tsup": "^8.3.5",

packages/synthetic-chain/src/cli/cli.ts

@@ -117,7 +117,7 @@ switch (cmd) {
         console.log(chalk.cyan.bold(`Testing ${proposal.proposalName}`));
         const image = imageNameForProposal(proposal, 'test');
         bakeTarget(image.target, values.dry);
-        runTestImage(proposal);
+        runTestImage({ proposal });
         // delete the image to reclaim disk space. The next build
         // will use the build cache.
         execSync('docker system df', { stdio: 'inherit' });

packages/synthetic-chain/src/cli/dockerfileGen.ts

@@ -32,7 +32,12 @@ ENV UPGRADE_TO=${to}
 RUN echo '. /usr/src/upgrade-test-scripts/env_setup.sh' >> ~/.bashrc
 
 # copy scripts
-COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_prepare_zero.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/env_setup.sh',
+  './upgrade-test-scripts/run_prepare_zero.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 SHELL ["/bin/bash", "-c"]
 # this is the only layer that starts ag0
 RUN /usr/src/upgrade-test-scripts/run_prepare_zero.sh
@@ -75,8 +80,18 @@ ENV \
     UPGRADE_INFO=${JSON.stringify(encodeUpgradeInfo(upgradeInfo))} \
     SKIP_PROPOSAL_VALIDATION=${skipProposalValidation}
 
-COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
-COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_prepare.sh ./upgrade-test-scripts/start_to_to.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  ['host', 'node_modules', 'test', 'test.sh'],
+  `./proposals/${path}`,
+  `/usr/src/proposals/${path}`,
+)}
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/env_setup.sh',
+  './upgrade-test-scripts/run_prepare.sh',
+  './upgrade-test-scripts/start_to_to.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 WORKDIR /usr/src/upgrade-test-scripts
 SHELL ["/bin/bash", "-c"]
 RUN ./run_prepare.sh ${path}
@@ -100,8 +115,19 @@ FROM ghcr.io/agoric/agoric-sdk:${sdkImageTag} as execute-${proposalName}
 WORKDIR /usr/src/upgrade-test-scripts
 
 # base is a fresh sdk image so set up the proposal and its dependencies
-COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
-COPY --link --chmod=755 ./upgrade-test-scripts/env_setup.sh ./upgrade-test-scripts/run_execute.sh  ./upgrade-test-scripts/start_to_to.sh ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  ['host', 'node_modules', 'test', 'test.sh'],
+  `./proposals/${path}`,
+  `/usr/src/proposals/${path}`,
+)}
+${createCopyCommand(
+  ['test.sh'],
+  './upgrade-test-scripts/env_setup.sh',
+  './upgrade-test-scripts/run_execute.sh',
+  './upgrade-test-scripts/start_to_to.sh',
+  './upgrade-test-scripts/install_deps.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${path}
 
 COPY --link --from=prepare-${proposalName} /root/.agoric /root/.agoric
@@ -122,15 +148,27 @@ RUN ./run_execute.sh ${planName}
 # EVAL ${proposalName}
 FROM use-${lastProposal.proposalName} as eval-${proposalName}
 
-COPY --exclude=host --exclude=test --exclude=test.sh --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
+${createCopyCommand(
+  ['host', 'node_modules', 'test', 'test.sh'],
+  `./proposals/${path}`,
+  `/usr/src/proposals/${path}`,
+)}
 
 WORKDIR /usr/src/upgrade-test-scripts
 
 # First stage of this proposal so install its deps.
-COPY --link ./upgrade-test-scripts/install_deps.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/install_deps.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 RUN --mount=type=cache,target=/root/.yarn ./install_deps.sh ${path}
 
-COPY --link --chmod=755 ./upgrade-test-scripts/*eval* /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/*eval*',
+  '/usr/src/upgrade-test-scripts/',
+)}
 SHELL ["/bin/bash", "-c"]
 RUN ./run_eval.sh ${path}
 `;
@@ -149,7 +187,12 @@ FROM ${previousStage}-${proposalName} as use-${proposalName}
 
 WORKDIR /usr/src/upgrade-test-scripts
 
-COPY --link --chmod=755 ./upgrade-test-scripts/run_use.sh ./upgrade-test-scripts/start_agd.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/run_use.sh',
+  './upgrade-test-scripts/start_agd.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 SHELL ["/bin/bash", "-c"]
 RUN ./run_use.sh ${path}
 ENTRYPOINT ./start_agd.sh
@@ -171,11 +214,19 @@ FROM use-${proposalName} as test-${proposalName}
 # Previous stages copied excluding test files (see COPY above). It would be good
 # to copy only missing files, but there may be none. Fortunately, copying extra
 # does not invalidate other images because nothing depends on this layer.
-COPY --exclude=host --link --chmod=755 ./proposals/${path} /usr/src/proposals/${path}
+${createCopyCommand(
+  ['host', 'node_modules'],
+  `./proposals/${path}`,
+  `/usr/src/proposals/${path}`,
+)}
 
 WORKDIR /usr/src/upgrade-test-scripts
 
-COPY --link --chmod=755 ./upgrade-test-scripts/run_test.sh /usr/src/upgrade-test-scripts/
+${createCopyCommand(
+  [],
+  './upgrade-test-scripts/run_test.sh',
+  '/usr/src/upgrade-test-scripts/',
+)}
 SHELL ["/bin/bash", "-c"]
 ENTRYPOINT ./run_test.sh ${path}
 `;
@@ -196,6 +247,18 @@ FROM ${useImage} as latest
   },
 };
 
+export const createCopyCommand = (
+  exclusionList: Array<string>,
+  ...files: Array<string>
+) =>
+  [
+    'COPY',
+    '--link',
+    '--chmod=755',
+    ...exclusionList.map(excluded => `--exclude=${excluded}`),
+    ...files,
+  ].join(' ');
+
 export function writeBakefileProposals(
   allProposals: ProposalInfo[],
   platforms?: Platform[],

packages/synthetic-chain/src/cli/run.ts

@@ -1,16 +1,14 @@
 import { spawnSync } from 'node:child_process';
 import { existsSync, realpathSync } from 'node:fs';
-import { resolve as resolvePath } from 'node:path';
+import { basename, resolve as resolvePath } from 'node:path';
+import { fileSync as createTempFile } from 'tmp';
 import { ProposalInfo, imageNameForProposal } from './proposals.js';
 
-const createMessageFile = () => {
-  const messageFileName = `${new Date().getTime()}.tmp`;
-  const messageFilePath = `/tmp/${messageFileName}`;
-  spawnSync('touch', [messageFilePath]);
-  return [messageFileName, messageFilePath];
-};
+const createMessageFile = (proposal: ProposalInfo) =>
+  createTempFile({ prefix: proposal.proposalName });
 
 const executeHostScriptIfPresent = (
+  extraEnv: typeof process.env,
   proposal: ProposalInfo,
   scriptName: string,
 ) => {
@@ -19,7 +17,10 @@ const executeHostScriptIfPresent = (
     console.log(
       `Running script ${scriptName} for proposal ${proposal.proposalName}`,
     );
-    spawnSync(scriptPath, { env: process.env, stdio: 'inherit' });
+    spawnSync(scriptPath, {
+      env: { ...process.env, ...extraEnv },
+      stdio: 'inherit',
+    });
   }
 };
 
@@ -45,77 +46,91 @@ const propagateSlogfile = env => {
   ];
 };
 
-export const runTestImage = (proposal: ProposalInfo) => {
-  const [messageFileName, messageFilePath] = createMessageFile();
+export const runTestImage = ({
+  extraDockerArgs = [],
+  proposal,
+  removeContainerOnExit = true,
+}: {
+  extraDockerArgs?: Array<string>;
+  proposal: ProposalInfo;
+  removeContainerOnExit?: boolean;
+}) => {
+  const { name: messageFilePath, removeCallback: removeTempFileCallback } =
+    createMessageFile(proposal);
+  const messageFileName = basename(messageFilePath);
 
   const containerFilePath = `/root/${messageFileName}`;
-  process.env.MESSAGE_FILE_PATH = messageFilePath;
-
-  executeHostScriptIfPresent(proposal, 'before-test-run.sh');
 
-  console.log(`Running test image for proposal ${proposal.proposalName}`);
-  const { name } = imageNameForProposal(proposal, 'test');
-  spawnSync(
-    'docker',
-    [
-      'run',
-      '--env',
-      `MESSAGE_FILE_PATH=${containerFilePath}`,
-      '--mount',
-      `source=${messageFilePath},target=${containerFilePath},type=bind`,
-      '--network',
-      'host',
-      '--rm',
-      ...propagateSlogfile(process.env),
-      name,
-    ],
-    { stdio: 'inherit' },
-  );
+  try {
+    executeHostScriptIfPresent(
+      {
+        MESSAGE_FILE_PATH: messageFilePath,
+      },
+      proposal,
+      'before-test-run.sh',
+    );
 
-  spawnSync('rm', ['--force', messageFilePath]);
+    console.log(`Running test image for proposal ${proposal.proposalName}`);
+    const { name } = imageNameForProposal(proposal, 'test');
+    spawnSync(
+      'docker',
+      [
+        'run',
+        '--env',
+        `MESSAGE_FILE_PATH=${containerFilePath}`,
+        '--mount',
+        `source=${messageFilePath},target=${containerFilePath},type=bind`,
+        '--network',
+        'host',
+        removeContainerOnExit && '--rm',
+        ...propagateSlogfile(process.env),
+        ...extraDockerArgs,
+        name,
+      ]
+        .filter(Boolean)
+        .map(String),
+      { stdio: 'inherit' },
+    );
 
-  executeHostScriptIfPresent(proposal, 'after-test-run.sh');
+    executeHostScriptIfPresent(
+      {
+        MESSAGE_FILE_PATH: messageFilePath,
+      },
+      proposal,
+      'after-test-run.sh',
+    );
+  } catch (err) {
+    removeTempFileCallback();
+    throw err;
+  }
 };
 
 export const debugTestImage = (proposal: ProposalInfo) => {
-  executeHostScriptIfPresent(proposal, 'before-test-run.sh');
   const { name } = imageNameForProposal(proposal, 'test');
   console.log(
     `
   Starting chain of test image for proposal ${proposal.proposalName}
-  
+
   To get an interactive shell in the container, use an IDE feature like "Attach Shell" or this command:'
-  
+
     docker exec -ti $(docker ps -q -f ancestor=${name}) bash
-  
+
   And within that shell:
     cd /usr/src/proposals/${proposal.path} && ./test.sh
-  
+
   To edit files you can use terminal tools like vim, or mount the container in your IDE.
   In VS Code the command is:
     Dev Containers: Attach to Running Container...
   `,
   );
-
-  // start the chain with ports mapped
-  spawnSync(
-    'docker',
-    [
-      'run',
+  return runTestImage({
+    extraDockerArgs: [
       '--entrypoint',
       '/usr/src/upgrade-test-scripts/start_agd.sh',
       '--interactive',
-      '--publish',
-      '1317:1317',
-      '--publish',
-      '9090:9090',
-      '--publish',
-      '26657:26657',
       '--tty',
-      ...propagateSlogfile(process.env),
-      name,
     ],
-    { stdio: 'inherit' },
-  );
-  executeHostScriptIfPresent(proposal, 'after-test-run.sh');
+    proposal,
+    removeContainerOnExit: false,
+  });
 };

yarn.lock

@@ -26,12 +26,14 @@ __metadata:
     "@types/better-sqlite3": "npm:^7.6.11"
     "@types/glob": "npm:^8.1.0"
     "@types/node": "npm:^18.19.50"
+    "@types/tmp": "npm:0.2.6"
     ava: "npm:^6.2.0"
     better-sqlite3: "npm:^11.8.0"
     chalk: "npm:^5.4.1"
     cosmjs-types: "npm:^0.9.0"
     execa: "npm:^9.5.2"
     glob: "npm:^11.0.0"
+    tmp: "npm:0.2.3"
     ts-blank-space: "npm:^0.5.0"
     tsup: "npm:^8.3.5"
     typescript: "npm:^5.7.3"
@@ -634,6 +636,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/tmp@npm:0.2.6":
+  version: 0.2.6
+  resolution: "@types/tmp@npm:0.2.6"
+  checksum: 10c0/a11bfa2cd8eaa6c5d62f62a3569192d7a2c28efdc5c17af0b0551db85816b2afc8156f3ca15ac76f0b142ae1403f04f44279871424233a1f3390b2e5fc828cd0
+  languageName: node
+  linkType: hard
+
 "@vercel/nft@npm:^0.27.5":
   version: 0.27.10
   resolution: "@vercel/nft@npm:0.27.10"
@@ -3326,6 +3335,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tmp@npm:0.2.3":
+  version: 0.2.3
+  resolution: "tmp@npm:0.2.3"
+  checksum: 10c0/3e809d9c2f46817475b452725c2aaa5d11985cf18d32a7a970ff25b568438e2c076c2e8609224feef3b7923fa9749b74428e3e634f6b8e520c534eef2fd24125
+  languageName: node
+  linkType: hard
+
 "to-regex-range@npm:^5.0.1":
   version: 5.0.1
   resolution: "to-regex-range@npm:5.0.1"