Merge pull request #150 from actions/sgoedecke/mock-inference-in-ci

Mock inference in CI

.github/workflows/ci.yml

@@ -56,20 +56,51 @@ jobs:
         id: checkout
         uses: actions/checkout@v5
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+
+      - name: Start Mock Inference Server
+        id: mock-server
+        run: |
+          node script/mock-inference-server.mjs &
+          echo "pid=$!" >> $GITHUB_OUTPUT
+          # Wait for server to be ready
+          for i in {1..10}; do
+            if curl -s http://localhost:3456/health > /dev/null; then
+              echo "Mock server is ready"
+              break
+            fi
+            sleep 1
+          done
+
       - name: Test Local Action
         id: test-action
-        continue-on-error: true
         uses: ./
         with:
           prompt: hello
+          endpoint: http://localhost:3456
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
       - name: Print Output
         id: output
-        continue-on-error: true
         run: echo "${{ steps.test-action.outputs.response }}"
 
+      - name: Verify Output
+        run: |
+          response="${{ steps.test-action.outputs.response }}"
+          if [[ -z "$response" ]]; then
+            echo "Error: No response received"
+            exit 1
+          fi
+          echo "Response received: $response"
+
+      - name: Stop Mock Server
+        if: always()
+        run: kill ${{ steps.mock-server.outputs.pid }} || true
+
   test-action-prompt-file:
     name: GitHub Actions Test with Prompt File
     runs-on: ubuntu-latest
@@ -79,6 +110,25 @@ jobs:
         id: checkout
         uses: actions/checkout@v5
 
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: .node-version
+
+      - name: Start Mock Inference Server
+        id: mock-server
+        run: |
+          node script/mock-inference-server.mjs &
+          echo "pid=$!" >> $GITHUB_OUTPUT
+          # Wait for server to be ready
+          for i in {1..10}; do
+            if curl -s http://localhost:3456/health > /dev/null; then
+              echo "Mock server is ready"
+              break
+            fi
+            sleep 1
+          done
+
       - name: Create Prompt File
         run: echo "hello" > prompt.txt
 
@@ -87,16 +137,33 @@ jobs:
 
       - name: Test Local Action with Prompt File
         id: test-action-prompt-file
-        continue-on-error: true
         uses: ./
         with:
           prompt-file: prompt.txt
           system-prompt-file: system-prompt.txt
+          endpoint: http://localhost:3456
         env:
           GITHUB_TOKEN: ${{ github.token }}
 
       - name: Print Output
-        continue-on-error: true
         run: |
           echo "Response saved to: ${{ steps.test-action-prompt-file.outputs.response-file }}"
           cat "${{ steps.test-action-prompt-file.outputs.response-file }}"
+
+      - name: Verify Output
+        run: |
+          response_file="${{ steps.test-action-prompt-file.outputs.response-file }}"
+          if [[ ! -f "$response_file" ]]; then
+            echo "Error: Response file not found"
+            exit 1
+          fi
+          content=$(cat "$response_file")
+          if [[ -z "$content" ]]; then
+            echo "Error: Response file is empty"
+            exit 1
+          fi
+          echo "Response file content: $content"
+
+      - name: Stop Mock Server
+        if: always()
+        run: kill ${{ steps.mock-server.outputs.pid }} || true

__tests__/main.test.ts

@@ -75,17 +75,13 @@ vi.mock('fs', () => ({
   writeFileSync: mockWriteFileSync,
 }))
 
-// Mocks for tmp module to control temporary file creation and cleanup
-const mockRemoveCallback = vi.fn()
+// Mocks for tmp module to control temporary file creation
 const mockFileSync = vi.fn().mockReturnValue({
   name: '/secure/temp/dir/modelResponse-abc123.txt',
-  removeCallback: mockRemoveCallback,
 })
-const mockSetGracefulCleanup = vi.fn()
 
 vi.mock('tmp', () => ({
   fileSync: mockFileSync,
-  setGracefulCleanup: mockSetGracefulCleanup,
 }))
 
 // Mock MCP and inference modules
@@ -283,7 +279,7 @@ describe('main.ts', () => {
     expect(mockProcessExit).toHaveBeenCalledWith(1)
   })
 
-  it('creates secure temporary files with proper cleanup', async () => {
+  it('creates temporary files that persist for downstream steps', async () => {
     mockInputs({
       prompt: 'Test prompt',
       'system-prompt': 'You are a test assistant.',
@@ -291,34 +287,16 @@ describe('main.ts', () => {
 
     await run()
 
-    expect(mockSetGracefulCleanup).toHaveBeenCalledOnce()
-
+    // Verify temp file is created with keep: true so it persists
     expect(mockFileSync).toHaveBeenCalledWith({
       prefix: 'modelResponse-',
       postfix: '.txt',
+      keep: true,
     })
 
     expect(core.setOutput).toHaveBeenNthCalledWith(2, 'response-file', '/secure/temp/dir/modelResponse-abc123.txt')
     expect(mockWriteFileSync).toHaveBeenCalledWith('/secure/temp/dir/modelResponse-abc123.txt', 'Hello, user!', 'utf-8')
-    expect(mockRemoveCallback).toHaveBeenCalledOnce()
 
     expect(mockProcessExit).toHaveBeenCalledWith(0)
   })
-
-  it('handles cleanup errors gracefully', async () => {
-    mockRemoveCallback.mockImplementationOnce(() => {
-      throw new Error('Cleanup failed')
-    })
-
-    mockInputs({
-      prompt: 'Test prompt',
-      'system-prompt': 'You are a test assistant.',
-    })
-
-    await run()
-
-    expect(mockRemoveCallback).toHaveBeenCalledOnce()
-    expect(core.warning).toHaveBeenCalledWith('Failed to cleanup temporary file: Error: Cleanup failed')
-    expect(mockProcessExit).toHaveBeenCalledWith(0)
-  })
 })

dist/index.js

@@ -52618,9 +52618,6 @@ function isPromptYamlFile(filePath) {
  * @returns Resolves when the action is complete.
  */
 async function run() {
-    let responseFile = null;
-    // Set up graceful cleanup for temporary files on process exit
-    tmpExports.setGracefulCleanup();
     try {
         const promptFilePath = coreExports.getInput('prompt-file');
         const inputVariables = coreExports.getInput('input');
@@ -52675,10 +52672,13 @@ async function run() {
             modelResponse = await simpleInference(inferenceRequest);
         }
         coreExports.setOutput('response', modelResponse || '');
-        // Create a secure temporary file instead of using the temp directory directly
-        responseFile = tmpExports.fileSync({
+        // Create a temporary file for the response that persists for downstream steps.
+        // We use keep: true to prevent automatic cleanup - the file will be cleaned up
+        // by the runner when the job completes.
+        const responseFile = tmpExports.fileSync({
             prefix: 'modelResponse-',
             postfix: '.txt',
+            keep: true,
         });
         coreExports.setOutput('response-file', responseFile.name);
         if (modelResponse && modelResponse !== '') {
@@ -52695,18 +52695,6 @@ async function run() {
         // Force exit to prevent hanging on open connections
         process.exit(1);
     }
-    finally {
-        // Explicit cleanup of temporary file if it was created
-        if (responseFile) {
-            try {
-                responseFile.removeCallback();
-            }
-            catch (cleanupError) {
-                // Log cleanup errors but don't fail the action
-                coreExports.warning(`Failed to cleanup temporary file: ${cleanupError}`);
-            }
-        }
-    }
     // Force exit to prevent hanging on open connections
     process.exit(0);
 }

eslint.config.mjs

@@ -19,7 +19,7 @@ const compat = new FlatCompat({
 
 export default [
   {
-    ignores: ['**/coverage', '**/dist', '**/linter', '**/node_modules'],
+    ignores: ['**/coverage', '**/dist', '**/linter', '**/node_modules', 'script/**'],
   },
   ...compat.extends(
     'eslint:recommended',

script/mock-inference-server.mjs

@@ -0,0 +1,71 @@
+#!/usr/bin/env node
+/**
+ * A simple mock OpenAI-compatible inference server for CI testing.
+ * This returns predictable responses without needing real API credentials.
+ */
+
+import http from 'http'
+
+const PORT = process.env.MOCK_SERVER_PORT || 3456
+
+const server = http.createServer((req, res) => {
+  let body = ''
+
+  req.on('data', chunk => {
+    body += chunk.toString()
+  })
+
+  req.on('end', () => {
+    console.log(`[Mock Server] ${req.method} ${req.url}`)
+
+    // Handle chat completions endpoint
+    if (req.url === '/chat/completions' && req.method === 'POST') {
+      const request = JSON.parse(body)
+      const userMessage = request.messages?.find(m => m.role === 'user')?.content || 'No prompt'
+
+      const response = {
+        id: 'mock-completion-id',
+        object: 'chat.completion',
+        created: Date.now(),
+        model: request.model || 'mock-model',
+        choices: [
+          {
+            index: 0,
+            message: {
+              role: 'assistant',
+              content: `Mock response to: "${userMessage.slice(0, 50)}..."`,
+            },
+            finish_reason: 'stop',
+          },
+        ],
+        usage: {
+          prompt_tokens: 10,
+          completion_tokens: 20,
+          total_tokens: 30,
+        },
+      }
+
+      res.writeHead(200, {'Content-Type': 'application/json'})
+      res.end(JSON.stringify(response))
+      return
+    }
+
+    // Health check endpoint
+    if (req.url === '/health' || req.url === '/') {
+      res.writeHead(200, {'Content-Type': 'application/json'})
+      res.end(JSON.stringify({status: 'ok'}))
+      return
+    }
+
+    // 404 for unknown routes
+    res.writeHead(404, {'Content-Type': 'application/json'})
+    res.end(JSON.stringify({error: 'Not found'}))
+  })
+})
+
+server.listen(PORT, () => {
+  console.log(`[Mock Server] Listening on http://localhost:${PORT}`)
+  console.log('[Mock Server] Endpoints:')
+  console.log('  POST /chat/completions - Mock chat completion')
+  console.log('  GET  /health           - Health check')
+})

src/main.ts

@@ -18,11 +18,6 @@ import {
  * @returns Resolves when the action is complete.
  */
 export async function run(): Promise<void> {
-  let responseFile: tmp.FileResult | null = null
-
-  // Set up graceful cleanup for temporary files on process exit
-  tmp.setGracefulCleanup()
-
   try {
     const promptFilePath = core.getInput('prompt-file')
     const inputVariables = core.getInput('input')
@@ -101,10 +96,13 @@ export async function run(): Promise<void> {
 
     core.setOutput('response', modelResponse || '')
 
-    // Create a secure temporary file instead of using the temp directory directly
-    responseFile = tmp.fileSync({
+    // Create a temporary file for the response that persists for downstream steps.
+    // We use keep: true to prevent automatic cleanup - the file will be cleaned up
+    // by the runner when the job completes.
+    const responseFile = tmp.fileSync({
       prefix: 'modelResponse-',
       postfix: '.txt',
+      keep: true,
     })
 
     core.setOutput('response-file', responseFile.name)
@@ -120,16 +118,6 @@ export async function run(): Promise<void> {
     }
     // Force exit to prevent hanging on open connections
     process.exit(1)
-  } finally {
-    // Explicit cleanup of temporary file if it was created
-    if (responseFile) {
-      try {
-        responseFile.removeCallback()
-      } catch (cleanupError) {
-        // Log cleanup errors but don't fail the action
-        core.warning(`Failed to cleanup temporary file: ${cleanupError}`)
-      }
-    }
   }
 
   // Force exit to prevent hanging on open connections