From 37331eae64817227bb2dc3c18204a9f21ced7c6f Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 28 Jul 2026 21:32:47 +0800 Subject: [PATCH 1/4] fix(agent): include select option labels and field name in autofill schema --- packages/agent/src/activities/agent.ts | 2 +- packages/agent/src/index.ts | 42 +++++++++++++++---- packages/agent/src/schema-conversion.test.ts | 43 ++++++++++++++++++-- packages/core/src/upload/upload.ts | 8 +--- 4 files changed, 76 insertions(+), 19 deletions(-) diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts index da83e722..dbe59f8f 100644 --- a/packages/agent/src/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -13,8 +13,8 @@ import { s3Service } from '@shumai/core/src/s3/s3' import { uploadService } from '@shumai/core/src/upload/upload' import { VersionStackService } from '@shumai/core/src/versionStack/versionStack' import { AssetType, prisma, Prisma, type Skill } from '@shumai/db' -import { ApplicationFailure, Context } from '@temporalio/activity' import { registerLocalCancelHandler, unregisterLocalCancelHandler } from '@shumai/workflow-core' +import { ApplicationFailure, Context } from '@temporalio/activity' import { generateKeyBetween } from 'jittered-fractional-indexing' import { ulid } from 'ulid' import { DatabaseSessionStorage } from '../database-session-storage' diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 2cdb9ba7..0fe3cef4 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -320,29 +320,55 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { const properties: Record = {} for (const f of fields) { let schema: TSchema + const fieldName = f.config.name switch (f.config.type) { case 'text': case 'longText': - schema = Type.String() + schema = Type.String({ title: fieldName }) break case 'number': case 'rating': - schema = Type.Number() + schema = Type.Number({ title: fieldName }) break case 'toggle': - schema = Type.Boolean() + schema = Type.Boolean({ title: fieldName }) break - case 'select': + case 'select': { + const options = f.config.select?.options || [] schema = Type.String({ - enum: f.config.select?.options?.map((o) => o.id) || [], + title: fieldName, + enum: options.map((o) => o.id), }) + if (options.length > 0) { + const optionMapStr = options.map((o) => `"${o.id}" (${o.displayName})`).join(', ') + const optionDesc = `[Allowed options: ${optionMapStr}]` + const baseDesc = f.description ? `${fieldName}: ${f.description}` : fieldName + schema.description = `${baseDesc} ${optionDesc}` + } break + } + case 'selectMulti': { + const options = f.config.selectMulti?.options || [] + schema = Type.Array( + Type.String({ + enum: options.map((o) => o.id), + }), + { title: fieldName }, + ) + if (options.length > 0) { + const optionMapStr = options.map((o) => `"${o.id}" (${o.displayName})`).join(', ') + const optionDesc = `[Allowed options: ${optionMapStr}]` + const baseDesc = f.description ? `${fieldName}: ${f.description}` : fieldName + schema.description = `${baseDesc} ${optionDesc}` + } + break + } default: - schema = Type.String() + schema = Type.String({ title: fieldName }) } - if (f.description) { - schema.description = f.description + if (!schema.description) { + schema.description = f.description ? `${fieldName}: ${f.description}` : fieldName } properties[f.id] = schema } diff --git a/packages/agent/src/schema-conversion.test.ts b/packages/agent/src/schema-conversion.test.ts index 898c8da5..48d7ceec 100644 --- a/packages/agent/src/schema-conversion.test.ts +++ b/packages/agent/src/schema-conversion.test.ts @@ -9,8 +9,11 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') - expect(schema.properties.f1.description).toBe('The title') + expect(schema.properties.f1.title).toBe('Title') + expect(schema.properties.f1.description).toBe('Title: The title') expect(schema.properties.f2.type).toBe('string') + expect(schema.properties.f2.title).toBe('Desc') + expect(schema.properties.f2.description).toBe('Desc') }) it('converts numeric fields', () => { @@ -20,21 +23,25 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('number') + expect(schema.properties.f1.title).toBe('Price') expect(schema.properties.f2.type).toBe('number') + expect(schema.properties.f2.title).toBe('Rating') }) it('converts toggle fields', () => { const fields: AutofillField[] = [{ id: 'f1', config: { name: 'Active', type: 'toggle' } }] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('boolean') + expect(schema.properties.f1.title).toBe('Active') }) - it('converts select fields with enums', () => { + it('converts select fields with enums and option descriptions', () => { const fields: AutofillField[] = [ { id: 'f1', + description: 'Species of animal', config: { - name: 'Status', + name: 'Species', type: 'select', select: { options: [ @@ -47,7 +54,36 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') + expect(schema.properties.f1.title).toBe('Species') expect(schema.properties.f1.enum).toEqual(['opt1', 'opt2']) + expect(schema.properties.f1.description).toBe( + 'Species: Species of animal [Allowed options: "opt1" (Option 1), "opt2" (Option 2)]', + ) + }) + + it('converts selectMulti fields with array enums and option descriptions', () => { + const fields: AutofillField[] = [ + { + id: 'f1', + config: { + name: 'Tags', + type: 'selectMulti', + selectMulti: { + options: [ + { id: 'opt1', displayName: 'Tag 1', color: 'red' }, + { id: 'opt2', displayName: 'Tag 2', color: 'blue' }, + ], + }, + }, + }, + ] + const schema = fieldsToTypeBoxSchema(fields) + expect(schema.properties.f1.type).toBe('array') + expect(schema.properties.f1.title).toBe('Tags') + expect(schema.properties.f1.items.enum).toEqual(['opt1', 'opt2']) + expect(schema.properties.f1.description).toBe( + 'Tags [Allowed options: "opt1" (Tag 1), "opt2" (Tag 2)]', + ) }) it('defaults to string for unknown types', () => { @@ -57,5 +93,6 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') + expect(schema.properties.f1.title).toBe('Unknown') }) }) diff --git a/packages/core/src/upload/upload.ts b/packages/core/src/upload/upload.ts index a7595c9c..9df9eb86 100644 --- a/packages/core/src/upload/upload.ts +++ b/packages/core/src/upload/upload.ts @@ -1,13 +1,7 @@ import { assetService } from '@shumai/core/src/asset/asset' import { PaginatedData, PaginationParams, paginateQuery } from '@shumai/core/src/pagination' import { s3Service } from '@shumai/core/src/s3/s3' -import { - AssetStatus, - AssetType, - Prisma, - TaskStatus, - prisma, -} from '@shumai/db' +import { AssetStatus, AssetType, Prisma, TaskStatus, prisma } from '@shumai/db' import { ConfirmFileUploadRequest, CreateUploadTaskRequest, From d6a74b82943fd275ba13661d72bf4b9252a5062d Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 28 Jul 2026 21:42:06 +0800 Subject: [PATCH 2/4] fix(agent): remove title property options from autofill schema for maximum provider compatibility --- packages/agent/src/activities/agent.ts | 1 + packages/agent/src/index.ts | 10 ++++------ packages/agent/src/schema-conversion.test.ts | 8 -------- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts index dbe59f8f..c5dbb016 100644 --- a/packages/agent/src/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -409,6 +409,7 @@ export interface AutofillAiParams { export async function autofillAiActivity(params: AutofillAiParams) { const prompt = 'Analyze the provided images and extract metadata.' const toolSchema = fieldsToTypeBoxSchema(params.fields) + console.log('==== autofill schema', toolSchema) let capturedData: Record | null = null const autofillTool: AgentTool = { diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 0fe3cef4..28ff9d2b 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -324,19 +324,18 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { switch (f.config.type) { case 'text': case 'longText': - schema = Type.String({ title: fieldName }) + schema = Type.String() break case 'number': case 'rating': - schema = Type.Number({ title: fieldName }) + schema = Type.Number() break case 'toggle': - schema = Type.Boolean({ title: fieldName }) + schema = Type.Boolean() break case 'select': { const options = f.config.select?.options || [] schema = Type.String({ - title: fieldName, enum: options.map((o) => o.id), }) if (options.length > 0) { @@ -353,7 +352,6 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { Type.String({ enum: options.map((o) => o.id), }), - { title: fieldName }, ) if (options.length > 0) { const optionMapStr = options.map((o) => `"${o.id}" (${o.displayName})`).join(', ') @@ -364,7 +362,7 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { break } default: - schema = Type.String({ title: fieldName }) + schema = Type.String() } if (!schema.description) { diff --git a/packages/agent/src/schema-conversion.test.ts b/packages/agent/src/schema-conversion.test.ts index 48d7ceec..320ec323 100644 --- a/packages/agent/src/schema-conversion.test.ts +++ b/packages/agent/src/schema-conversion.test.ts @@ -9,10 +9,8 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') - expect(schema.properties.f1.title).toBe('Title') expect(schema.properties.f1.description).toBe('Title: The title') expect(schema.properties.f2.type).toBe('string') - expect(schema.properties.f2.title).toBe('Desc') expect(schema.properties.f2.description).toBe('Desc') }) @@ -23,16 +21,13 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('number') - expect(schema.properties.f1.title).toBe('Price') expect(schema.properties.f2.type).toBe('number') - expect(schema.properties.f2.title).toBe('Rating') }) it('converts toggle fields', () => { const fields: AutofillField[] = [{ id: 'f1', config: { name: 'Active', type: 'toggle' } }] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('boolean') - expect(schema.properties.f1.title).toBe('Active') }) it('converts select fields with enums and option descriptions', () => { @@ -54,7 +49,6 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') - expect(schema.properties.f1.title).toBe('Species') expect(schema.properties.f1.enum).toEqual(['opt1', 'opt2']) expect(schema.properties.f1.description).toBe( 'Species: Species of animal [Allowed options: "opt1" (Option 1), "opt2" (Option 2)]', @@ -79,7 +73,6 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('array') - expect(schema.properties.f1.title).toBe('Tags') expect(schema.properties.f1.items.enum).toEqual(['opt1', 'opt2']) expect(schema.properties.f1.description).toBe( 'Tags [Allowed options: "opt1" (Tag 1), "opt2" (Tag 2)]', @@ -93,6 +86,5 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') - expect(schema.properties.f1.title).toBe('Unknown') }) }) From 19df011dc2df7eb2b6b1d2de59517a7ad6736b58 Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 28 Jul 2026 21:44:56 +0800 Subject: [PATCH 3/4] fix(agent): format autofill schema property descriptions with field name and available options list --- packages/agent/src/index.ts | 15 ++++++--------- packages/agent/src/schema-conversion.test.ts | 8 ++++---- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 28ff9d2b..4740676e 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -321,6 +321,7 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { for (const f of fields) { let schema: TSchema const fieldName = f.config.name + const fieldDesc = f.description || fieldName switch (f.config.type) { case 'text': case 'longText': @@ -339,10 +340,8 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { enum: options.map((o) => o.id), }) if (options.length > 0) { - const optionMapStr = options.map((o) => `"${o.id}" (${o.displayName})`).join(', ') - const optionDesc = `[Allowed options: ${optionMapStr}]` - const baseDesc = f.description ? `${fieldName}: ${f.description}` : fieldName - schema.description = `${baseDesc} ${optionDesc}` + const optionLines = options.map((o) => `- ${o.displayName} => ${o.id}`).join('\n') + schema.description = `The field '${fieldName}' represents ${fieldDesc}.\nSelect one option and return the option ID as the value.\n\nAvailable options:\n${optionLines}` } break } @@ -354,10 +353,8 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { }), ) if (options.length > 0) { - const optionMapStr = options.map((o) => `"${o.id}" (${o.displayName})`).join(', ') - const optionDesc = `[Allowed options: ${optionMapStr}]` - const baseDesc = f.description ? `${fieldName}: ${f.description}` : fieldName - schema.description = `${baseDesc} ${optionDesc}` + const optionLines = options.map((o) => `- ${o.displayName} => ${o.id}`).join('\n') + schema.description = `The field '${fieldName}' represents ${fieldDesc}.\nSelect applicable options and return the option IDs as the value.\n\nAvailable options:\n${optionLines}` } break } @@ -366,7 +363,7 @@ export function fieldsToTypeBoxSchema(fields: AutofillField[]) { } if (!schema.description) { - schema.description = f.description ? `${fieldName}: ${f.description}` : fieldName + schema.description = `The field '${fieldName}' represents ${fieldDesc}.` } properties[f.id] = schema } diff --git a/packages/agent/src/schema-conversion.test.ts b/packages/agent/src/schema-conversion.test.ts index 320ec323..9a101659 100644 --- a/packages/agent/src/schema-conversion.test.ts +++ b/packages/agent/src/schema-conversion.test.ts @@ -9,9 +9,9 @@ describe('fieldsToTypeBoxSchema', () => { ] const schema = fieldsToTypeBoxSchema(fields) expect(schema.properties.f1.type).toBe('string') - expect(schema.properties.f1.description).toBe('Title: The title') + expect(schema.properties.f1.description).toBe("The field 'Title' represents The title.") expect(schema.properties.f2.type).toBe('string') - expect(schema.properties.f2.description).toBe('Desc') + expect(schema.properties.f2.description).toBe("The field 'Desc' represents Desc.") }) it('converts numeric fields', () => { @@ -51,7 +51,7 @@ describe('fieldsToTypeBoxSchema', () => { expect(schema.properties.f1.type).toBe('string') expect(schema.properties.f1.enum).toEqual(['opt1', 'opt2']) expect(schema.properties.f1.description).toBe( - 'Species: Species of animal [Allowed options: "opt1" (Option 1), "opt2" (Option 2)]', + "The field 'Species' represents Species of animal.\nSelect one option and return the option ID as the value.\n\nAvailable options:\n- Option 1 => opt1\n- Option 2 => opt2", ) }) @@ -75,7 +75,7 @@ describe('fieldsToTypeBoxSchema', () => { expect(schema.properties.f1.type).toBe('array') expect(schema.properties.f1.items.enum).toEqual(['opt1', 'opt2']) expect(schema.properties.f1.description).toBe( - 'Tags [Allowed options: "opt1" (Tag 1), "opt2" (Tag 2)]', + "The field 'Tags' represents Tags.\nSelect applicable options and return the option IDs as the value.\n\nAvailable options:\n- Tag 1 => opt1\n- Tag 2 => opt2", ) }) From 006fb1726b5e0e54f3cc92da9c16bc3d6b4216e4 Mon Sep 17 00:00:00 2001 From: yiling Date: Tue, 28 Jul 2026 21:46:14 +0800 Subject: [PATCH 4/4] remove debug log --- packages/agent/src/activities/agent.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/agent/src/activities/agent.ts b/packages/agent/src/activities/agent.ts index c5dbb016..365d1092 100644 --- a/packages/agent/src/activities/agent.ts +++ b/packages/agent/src/activities/agent.ts @@ -1,7 +1,7 @@ import { - type AgentMessage, - type AgentTool, - type SessionTreeEntry, + type AgentMessage, + type AgentTool, + type SessionTreeEntry, } from '@earendil-works/pi-agent-core' import { type ImageContent } from '@earendil-works/pi-ai' import { assetService } from '@shumai/core/src/asset/asset' @@ -19,10 +19,10 @@ import { generateKeyBetween } from 'jittered-fractional-indexing' import { ulid } from 'ulid' import { DatabaseSessionStorage } from '../database-session-storage' import { - createAgentSession, - fieldsToTypeBoxSchema, - type AutofillField, - type DbProviderInfo, + createAgentSession, + fieldsToTypeBoxSchema, + type AutofillField, + type DbProviderInfo, } from '../index' import { aiUsageService } from '@shumai/core/src/ai-usage/ai-usage' @@ -409,7 +409,6 @@ export interface AutofillAiParams { export async function autofillAiActivity(params: AutofillAiParams) { const prompt = 'Analyze the provided images and extract metadata.' const toolSchema = fieldsToTypeBoxSchema(params.fields) - console.log('==== autofill schema', toolSchema) let capturedData: Record | null = null const autofillTool: AgentTool = {