import { expect } from 'vitest'; import path from 'path'; import { describeMdxFile, testAtlas, runCurlCommand, unpackComponents, parseJsonBlock, parseCurlCommand, } from '../../../tests/fixtures.js'; import { waitForTask, poll } from '../../../tests/atlas-utils.js'; const BASE_SOURCE = 'document-ocr'; const BASE_COLLECTION = 'scanned-documents'; const BASE_PARTITION = 'document-ocr-search'; const SAMPLE_TASK_ID = 'bk_4w2Fn0ZyxZVuiG1nS8Qi7v9UBc'; const SAMPLE_OBJECT_PATH = 'scanned-documents%2F0f6c31ec'; function applyReplacements(content, replacements) { let result = content; // Sort replacement keys by string length (longest first) to prevent partial replacements // when one replacement key is a substring of another. const ordered = [...replacements.entries()].sort((a, b) => b[0].length - a[0].length); for (const [needle, value] of ordered) { result = result.split(needle).join(value); } return result; } function extractDataPart(command) { const single = command.match(/-F 'data=([^']+)'/); if (single) return single[1]; const dbl = command.match(/-F "data=([^"]+)"/); if (dbl) return dbl[1]; throw new Error('Unable to locate data= form part in upload command'); } function extractFilePart(command) { const match = command.match(/-F\s+["']file=@([^;]+);type=([^"']+)["']/); if (!match) { throw new Error('Unable to locate file= form part in upload command'); } const fullPath = match[1]; const type = match[2]; const filename = fullPath.split('/').pop() || 'upload.bin'; return { filename, type }; } function createSamplePdf(text) { let pdf = '%PDF-1.4\n'; const offsets = [0]; const addObject = (id, body) => { offsets[id] = pdf.length; pdf += `${id} 0 obj\n${body}\nendobj\n`; }; addObject(1, '<< /Type /Catalog /Pages 2 0 R >>'); addObject(2, '<< /Type /Pages /Kids [3 0 R] /Count 1 >>'); addObject(3, '<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>'); const stream = `BT\n/F1 18 Tf\n72 720 Td\n(${text}) Tj\nET\n`; addObject(4, `<< /Length ${stream.length} >>\nstream\n${stream}endstream\n`); addObject(5, '<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'); const xrefOffset = pdf.length; pdf += 'xref\n'; pdf += '0 6\n'; pdf += '0000000000 65535 f \n'; for (let i = 1; i <= 5; i += 1) { pdf += `${offsets[i].toString().padStart(10, '0')} 00000 n \n`; } pdf += 'trailer\n'; pdf += '<< /Size 6 /Root 1 0 R >>\n'; pdf += 'startxref\n'; pdf += `${xrefOffset}\n`; pdf += '%%EOF\n'; return Buffer.from(pdf, 'ascii'); } async function runMultipartUpload(block, client, replacements) { const command = applyReplacements(block.props.content, replacements); const parsed = parseCurlCommand(command); const relativePath = parsed.url.split('$CLINIA_WORKSPACE')[1]; if (!relativePath) { throw new Error('Upload command missing $CLINIA_WORKSPACE placeholder'); } const headers = { ...parsed.headers }; delete headers['X-Clinia-API-Key']; const dataPart = extractDataPart(command); const filePart = extractFilePart(command); const formData = new FormData(); formData.set('data', dataPart); const pdfBuffer = createSamplePdf('Patient Name: Jane Doe; Date of Birth: 1984-03-17; Primary Care Physician: Example; 2024 Intake Form.'); const blob = new Blob([pdfBuffer], { type: filePart.type }); formData.set('file', blob, filePart.filename); return client.fetchAndAnnotate(relativePath, { method: parsed.method, headers, body: formData, }); } describeMdxFile(path.resolve(__dirname, 'file-ingestion-ocr.mdx'), async (components) => { const labels = [ 'ocrSource', 'ocrCollection', 'ocrPipeline', 'ocrUpload', 'ocrUploadResponse', 'ocrTask', 'ocrTaskResponse', 'ocrObject', 'ocrObjectResponse', 'ocrFileDownload', 'ocrPartition', 'ocrObjectSearch', ]; const [blocks, remaining] = unpackComponents(labels, components); const [ ocrSource, ocrCollection, ocrPipeline, ocrUpload, ocrUploadResponse, ocrTask, ocrTaskResponse, ocrObject, ocrObjectResponse, ocrFileDownload, ocrPartition, ocrObjectSearch, ] = blocks; // This test never passed yet. It still needs some work testAtlas.skip('File ingestion with OCR pipeline', async (client) => { const suffix = Math.floor(Math.random() * 10_000); const sourceKey = `${BASE_SOURCE}-${suffix}`; const collectionKey = `${BASE_COLLECTION}-${suffix}`; const partitionKey = `${BASE_PARTITION}-${suffix}`; const replacements = new Map([ [BASE_PARTITION, partitionKey], [BASE_COLLECTION, collectionKey], [BASE_SOURCE, sourceKey], ]); await client.delete(`/catalog/v1/partitions/${partitionKey}`); await client.delete(`/catalog/v1/sources/${sourceKey}`); let res = await runCurlCommand(applyReplacements(ocrSource.props.content, replacements), client); expect(res.ok, 'Failed to create OCR source').toBe(true); res = await runCurlCommand(applyReplacements(ocrCollection.props.content, replacements), client); expect(res.ok, 'Failed to create OCR collection').toBe(true); res = await runCurlCommand(applyReplacements(ocrPipeline.props.content, replacements), client); expect(res.ok, 'Failed to upsert OCR pipeline').toBe(true); res = await runMultipartUpload(ocrUpload, client, replacements); expect(res.ok, 'Failed to upload OCR object').toBe(true); const uploadBody = await res.json(); const expectedUpload = parseJsonBlock(ocrUploadResponse.props.content); expect(uploadBody.status).toBe(expectedUpload.status); const taskId = uploadBody.taskId; expect(typeof taskId).toBe('string'); const taskInfo = await waitForTask(client, taskId, sourceKey, { timeout: 15000 }); expect(taskInfo.status).toBe('SUCCESS'); const receipt = taskInfo.receipts?.find((r) => r?.target?.targetType === 'OBJECT'); expect(receipt, 'Task receipts should include created object').toBeTruthy(); const objectId = receipt.target.id; const taskCommand = applyReplacements(ocrTask.props.content, replacements).replace(SAMPLE_TASK_ID, taskId); res = await runCurlCommand(taskCommand, client); expect(res.ok, 'Failed to fetch OCR task via snippet').toBe(true); const taskSnapshot = await res.json(); expect(taskSnapshot.status).toBe('SUCCESS'); const expectedTask = parseJsonBlock(ocrTaskResponse.props.content); const snapshotReceipt = taskSnapshot.receipts.find((r) => r?.target?.targetType === expectedTask.receipts[0].target.targetType); expect(snapshotReceipt).toBeTruthy(); const objectCommand = applyReplacements(ocrObject.props.content, replacements).replace(SAMPLE_OBJECT_PATH, encodeURIComponent(objectId)); res = await runCurlCommand(objectCommand, client); expect(res.ok, 'Failed to fetch OCR object').toBe(true); const objectData = await res.json(); expect(objectData.meta.source).toBe(sourceKey); expect(Array.isArray(objectData.data.tags)).toBe(true); expect(objectData.data.tags).toContain('intake'); expect(typeof objectData.data.content).toBe('string'); expect(objectData.data.content.toLowerCase()).toContain('jane doe'); const expectedObject = parseJsonBlock(ocrObjectResponse.props.content); expect(objectData.data.tags.sort()).toEqual(expectedObject.data.tags.sort()); const fileDownloadCommand = applyReplacements(ocrFileDownload.props.content, replacements).replace(SAMPLE_OBJECT_PATH, encodeURIComponent(objectId)); res = await runCurlCommand(fileDownloadCommand, client); expect(res.ok, 'Failed to download OCR object file').toBe(true); const downloadType = res.headers.get('content-type') || ''; expect(downloadType.includes('application/')).toBe(true); res = await runCurlCommand(applyReplacements(ocrPartition.props.content, replacements), client); expect(res.ok, 'Failed to link OCR partition').toBe(true); const searchResult = await poll(async () => { const searchCommand = applyReplacements(ocrObjectSearch.props.content, replacements); const searchRes = await runCurlCommand(searchCommand, client); expect(searchRes.ok, 'Search request failed').toBe(true); const payload = await searchRes.json(); if (payload.meta.total > 0) { return payload; } return 'retry'; }, { timeout: 8000 }); expect(searchResult.meta.total).toBeGreaterThan(0); }); expect( remaining.length, `Unhandled blocks: ${remaining.map((c) => ` ${c.sourceFile}:${c.lineNumber} `)}`, ).toBe(0); });

About Clinia

Core Concepts

Search

Configuring Data Sources

Configuring Partitions

Managing Data

Master Data Management

Terminology

Identity and Access Management

Agents

Page Not Found