test: stabilize installed skill e2e harness
This commit is contained in:
Haitao Pan
parent
a6e059bca7
commit
6ee84b4e8e
114
test/features/assistant_page_installed_skill_e2e_suite.dart
Normal file
114
test/features/assistant_page_installed_skill_e2e_suite.dart
Normal file
@ -0,0 +1,114 @@
|
||||
// ignore_for_file: unused_import, unnecessary_import
|
||||
|
||||
@TestOn('vm')
|
||||
library;
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_test/flutter_test.dart';
|
||||
import 'package:xworkmate/app/app_controller.dart';
|
||||
import 'package:xworkmate/app/app_controller_desktop_thread_sessions.dart';
|
||||
import 'package:xworkmate/app/app_controller_desktop_workspace_execution.dart';
|
||||
import 'package:xworkmate/runtime/desktop_thread_artifact_service.dart';
|
||||
import 'package:xworkmate/runtime/runtime_models.dart';
|
||||
|
||||
import 'assistant_page_suite_support.dart';
|
||||
|
||||
void main() {
|
||||
group('AssistantPage installed skill E2E harness', () {
|
||||
for (final testCase in installedSkillE2ECasesInternal) {
|
||||
test('discovers, binds, and handoffs ${testCase.skillKey}', () async {
|
||||
final tempDirectory = await Directory.systemTemp.createTemp(
|
||||
'xworkmate-installed-skill-${testCase.skillKey}-',
|
||||
);
|
||||
addTearDown(() async {
|
||||
if (await tempDirectory.exists()) {
|
||||
try {
|
||||
await tempDirectory.delete(recursive: true);
|
||||
} catch (_) {}
|
||||
}
|
||||
});
|
||||
|
||||
final skillsRoot = Directory('${tempDirectory.path}/installed-skills');
|
||||
final workspaceRoot = Directory('${tempDirectory.path}/workspace');
|
||||
await workspaceRoot.create(recursive: true);
|
||||
|
||||
await writeSkillInternal(
|
||||
skillsRoot,
|
||||
'pptx',
|
||||
skillName: 'pptx',
|
||||
description: 'Presentation creation, editing, and QA.',
|
||||
);
|
||||
await writeSkillInternal(
|
||||
skillsRoot,
|
||||
'docx',
|
||||
skillName: 'docx',
|
||||
description: 'Word document authoring and editing.',
|
||||
);
|
||||
await writeSkillInternal(
|
||||
skillsRoot,
|
||||
'xlsx',
|
||||
skillName: 'xlsx',
|
||||
description: 'Spreadsheet authoring and formula validation.',
|
||||
);
|
||||
await writeSkillInternal(
|
||||
skillsRoot,
|
||||
'pdf',
|
||||
skillName: 'pdf',
|
||||
description: 'PDF extraction, creation, and form workflows.',
|
||||
);
|
||||
|
||||
final controller = await createInstalledSkillE2EControllerSimpleInternal(
|
||||
tempDirectory: tempDirectory,
|
||||
skillsRoot: skillsRoot,
|
||||
workspaceRoot: workspaceRoot,
|
||||
testCase: testCase,
|
||||
);
|
||||
|
||||
final sendFuture = controller.sendChatMessage(
|
||||
testCase.prompt,
|
||||
selectedSkillLabels: <String>[testCase.skillKey],
|
||||
);
|
||||
await waitForConditionInternal(() => controller.sendCallCount == 1);
|
||||
|
||||
expect(controller.lastSentMessage, contains(testCase.prompt));
|
||||
expect(controller.lastPromptInternal, contains(testCase.prompt));
|
||||
expect(
|
||||
controller.lastSelectedSkillLabelsInternal,
|
||||
equals(<String>[testCase.skillKey]),
|
||||
);
|
||||
expect(controller.lastWorkspacePathInternal, isNotEmpty);
|
||||
|
||||
controller.sendGate.complete();
|
||||
await sendFuture;
|
||||
|
||||
final artifactService = DesktopThreadArtifactService();
|
||||
final snapshot = await artifactService.loadSnapshot(
|
||||
workspacePath: controller.lastWorkspacePathInternal,
|
||||
workspaceKind: WorkspaceRefKind.localPath,
|
||||
);
|
||||
|
||||
expect(
|
||||
snapshot.fileEntries.map((item) => item.relativePath),
|
||||
contains(testCase.outputRelativePath),
|
||||
);
|
||||
expect(
|
||||
snapshot.resultEntries.map((item) => item.relativePath),
|
||||
contains(testCase.outputRelativePath),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test('records deferred media skill coverage explicitly', () {
|
||||
expect(
|
||||
installedSkillE2EDeferredCoverageInternal,
|
||||
equals(const <String>[
|
||||
'image-cog',
|
||||
'wan-image-video-generation-editting',
|
||||
'video-translator',
|
||||
'image-resizer',
|
||||
]),
|
||||
);
|
||||
}, skip: 'Deferred until the media skill packs are installed.');
|
||||
});
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
import '../test_suite_stub.dart'
|
||||
if (dart.library.io) 'assistant_page_installed_skill_e2e_suite.dart'
|
||||
as suite;
|
||||
|
||||
void main() {
|
||||
suite.main();
|
||||
}
|
||||
@ -22,6 +22,7 @@ import 'package:xworkmate/runtime/secure_config_store.dart';
|
||||
import 'package:xworkmate/theme/app_theme.dart';
|
||||
import 'package:xworkmate/widgets/pane_resize_handle.dart';
|
||||
import '../test_support.dart';
|
||||
import '../runtime/app_controller_thread_skills_suite_fixtures.dart';
|
||||
import 'assistant_page_suite_core.dart';
|
||||
import 'assistant_page_suite_composer.dart';
|
||||
|
||||
@ -179,12 +180,15 @@ class PendingSendAppControllerInternal extends AppController {
|
||||
PendingSendAppControllerInternal({
|
||||
required SecureConfigStore store,
|
||||
required this.sendGate,
|
||||
List<String>? singleAgentSharedSkillScanRootOverrides,
|
||||
}) : super(
|
||||
store: store,
|
||||
runtimeCoordinator: RuntimeCoordinator(
|
||||
gateway: FakeGatewayRuntimeInternal(store: store),
|
||||
codex: FakeCodexRuntimeInternal(),
|
||||
),
|
||||
singleAgentSharedSkillScanRootOverrides:
|
||||
singleAgentSharedSkillScanRootOverrides,
|
||||
);
|
||||
|
||||
final Completer<void> sendGate;
|
||||
@ -207,6 +211,215 @@ class PendingSendAppControllerInternal extends AppController {
|
||||
}
|
||||
}
|
||||
|
||||
class InstalledSkillE2ECaseInternal {
|
||||
const InstalledSkillE2ECaseInternal({
|
||||
required this.skillKey,
|
||||
required this.prompt,
|
||||
required this.outputRelativePath,
|
||||
required this.outputContent,
|
||||
});
|
||||
|
||||
final String skillKey;
|
||||
final String prompt;
|
||||
final String outputRelativePath;
|
||||
final String outputContent;
|
||||
}
|
||||
|
||||
const List<InstalledSkillE2ECaseInternal>
|
||||
installedSkillE2ECasesInternal = <InstalledSkillE2ECaseInternal>[
|
||||
InstalledSkillE2ECaseInternal(
|
||||
skillKey: 'pptx',
|
||||
prompt: 'Create a concise slide outline for the quarterly review.',
|
||||
outputRelativePath: 'artifacts/pptx/result.md',
|
||||
outputContent: '# pptx\n\nCaptured slide outline for the quarterly review.',
|
||||
),
|
||||
InstalledSkillE2ECaseInternal(
|
||||
skillKey: 'docx',
|
||||
prompt: 'Draft a short policy note with headings and bullets.',
|
||||
outputRelativePath: 'artifacts/docx/result.md',
|
||||
outputContent: '# docx\n\nCaptured policy note with headings and bullets.',
|
||||
),
|
||||
InstalledSkillE2ECaseInternal(
|
||||
skillKey: 'xlsx',
|
||||
prompt: 'Prepare a tiny table with one formula and one formatted cell.',
|
||||
outputRelativePath: 'artifacts/xlsx/result.md',
|
||||
outputContent: '# xlsx\n\nCaptured spreadsheet result with formula notes.',
|
||||
),
|
||||
InstalledSkillE2ECaseInternal(
|
||||
skillKey: 'pdf',
|
||||
prompt: 'Summarize a reference PDF and keep the output deterministic.',
|
||||
outputRelativePath: 'artifacts/pdf/result.md',
|
||||
outputContent: '# pdf\n\nCaptured PDF summary output.',
|
||||
),
|
||||
];
|
||||
|
||||
const List<String> installedSkillE2EDeferredCoverageInternal = <String>[
|
||||
'image-cog',
|
||||
'wan-image-video-generation-editting',
|
||||
'video-translator',
|
||||
'image-resizer',
|
||||
];
|
||||
|
||||
class InstalledSkillE2EAppControllerInternal
|
||||
extends PendingSendAppControllerInternal {
|
||||
InstalledSkillE2EAppControllerInternal({
|
||||
required super.store,
|
||||
required super.sendGate,
|
||||
required this.outputRelativePath,
|
||||
required this.outputContent,
|
||||
required this.importedSkill,
|
||||
super.singleAgentSharedSkillScanRootOverrides,
|
||||
this.sessionKey = 'installed-skill-session',
|
||||
});
|
||||
|
||||
final String outputRelativePath;
|
||||
final String outputContent;
|
||||
final AssistantThreadSkillEntry importedSkill;
|
||||
final String sessionKey;
|
||||
String lastPromptInternal = '';
|
||||
List<String> lastSelectedSkillLabelsInternal = const <String>[];
|
||||
String lastWorkspacePathInternal = '';
|
||||
|
||||
@override
|
||||
Future<void> sendChatMessage(
|
||||
String message, {
|
||||
String thinking = 'off',
|
||||
List<GatewayChatAttachmentPayload> attachments =
|
||||
const <GatewayChatAttachmentPayload>[],
|
||||
List<CollaborationAttachment> localAttachments =
|
||||
const <CollaborationAttachment>[],
|
||||
List<String> selectedSkillLabels = const <String>[],
|
||||
}) async {
|
||||
lastPromptInternal = message;
|
||||
lastSelectedSkillLabelsInternal = List<String>.unmodifiable(
|
||||
selectedSkillLabels,
|
||||
);
|
||||
lastWorkspacePathInternal = assistantWorkspacePathForSession(
|
||||
sessionKey,
|
||||
);
|
||||
final workspacePath = lastWorkspacePathInternal.trim();
|
||||
if (workspacePath.isNotEmpty) {
|
||||
final outputFile = File('$workspacePath/$outputRelativePath');
|
||||
await outputFile.parent.create(recursive: true);
|
||||
await outputFile.writeAsString(outputContent, flush: true);
|
||||
}
|
||||
await super.sendChatMessage(
|
||||
message,
|
||||
thinking: thinking,
|
||||
attachments: attachments,
|
||||
localAttachments: localAttachments,
|
||||
selectedSkillLabels: selectedSkillLabels,
|
||||
);
|
||||
}
|
||||
|
||||
@override
|
||||
String get currentSessionKey => sessionKey;
|
||||
}
|
||||
|
||||
Future<InstalledSkillE2EAppControllerInternal>
|
||||
createInstalledSkillE2EControllerInternal(
|
||||
WidgetTester tester, {
|
||||
required Directory tempDirectory,
|
||||
required Directory skillsRoot,
|
||||
required Directory workspaceRoot,
|
||||
required InstalledSkillE2ECaseInternal testCase,
|
||||
}) async {
|
||||
SharedPreferences.setMockInitialValues(<String, Object>{});
|
||||
print('installed-skill ${testCase.skillKey}: helper creating store');
|
||||
final store = SecureConfigStore(
|
||||
enableSecureStorage: false,
|
||||
databasePathResolver: () async => '${tempDirectory.path}/settings.db',
|
||||
fallbackDirectoryPathResolver: () async => tempDirectory.path,
|
||||
defaultSupportDirectoryPathResolver: () async => tempDirectory.path,
|
||||
);
|
||||
await store.initialize();
|
||||
await store.saveSettingsSnapshot(
|
||||
singleAgentTestSettingsInternal(workspacePath: workspaceRoot.path).copyWith(
|
||||
assistantExecutionTarget: AssistantExecutionTarget.singleAgent,
|
||||
multiAgent: MultiAgentConfig.defaults().copyWith(enabled: false),
|
||||
),
|
||||
);
|
||||
print('installed-skill ${testCase.skillKey}: helper creating controller');
|
||||
|
||||
final controller = InstalledSkillE2EAppControllerInternal(
|
||||
store: store,
|
||||
sendGate: Completer<void>(),
|
||||
outputRelativePath: testCase.outputRelativePath,
|
||||
outputContent: testCase.outputContent,
|
||||
importedSkill: AssistantThreadSkillEntry(
|
||||
key: testCase.skillKey,
|
||||
label: testCase.skillKey,
|
||||
description: 'Installed skill under test',
|
||||
sourcePath: '${skillsRoot.path}/${testCase.skillKey}',
|
||||
sourceLabel: testCase.skillKey,
|
||||
),
|
||||
singleAgentSharedSkillScanRootOverrides: <String>[skillsRoot.path],
|
||||
);
|
||||
print('installed-skill ${testCase.skillKey}: helper controller created');
|
||||
addTearDown(controller.dispose);
|
||||
print('installed-skill ${testCase.skillKey}: helper pumping once');
|
||||
await tester.pump(const Duration(milliseconds: 100));
|
||||
print('installed-skill ${testCase.skillKey}: helper pumped once');
|
||||
final stopwatch = Stopwatch()..start();
|
||||
while (controller.initializing) {
|
||||
print(
|
||||
'installed-skill ${testCase.skillKey}: helper waiting ${stopwatch.elapsedMilliseconds}ms',
|
||||
);
|
||||
if (stopwatch.elapsed > const Duration(seconds: 10)) {
|
||||
fail('controller did not finish initializing before timeout');
|
||||
}
|
||||
await tester.pump(const Duration(milliseconds: 20));
|
||||
}
|
||||
controller.upsertTaskThreadInternal(
|
||||
controller.currentSessionKey,
|
||||
importedSkills: <AssistantThreadSkillEntry>[controller.importedSkill],
|
||||
selectedSkillKeys: <String>[controller.importedSkill.key],
|
||||
);
|
||||
print('installed-skill ${testCase.skillKey}: helper initialized');
|
||||
return controller;
|
||||
}
|
||||
|
||||
Future<InstalledSkillE2EAppControllerInternal>
|
||||
createInstalledSkillE2EControllerSimpleInternal({
|
||||
required Directory tempDirectory,
|
||||
required Directory skillsRoot,
|
||||
required Directory workspaceRoot,
|
||||
required InstalledSkillE2ECaseInternal testCase,
|
||||
}) async {
|
||||
SharedPreferences.setMockInitialValues(<String, Object>{});
|
||||
final store = SecureConfigStore(
|
||||
enableSecureStorage: false,
|
||||
databasePathResolver: () async => '${tempDirectory.path}/settings.db',
|
||||
fallbackDirectoryPathResolver: () async => tempDirectory.path,
|
||||
defaultSupportDirectoryPathResolver: () async => tempDirectory.path,
|
||||
);
|
||||
await store.initialize();
|
||||
await store.saveSettingsSnapshot(
|
||||
singleAgentTestSettingsInternal(workspacePath: workspaceRoot.path).copyWith(
|
||||
assistantExecutionTarget: AssistantExecutionTarget.singleAgent,
|
||||
multiAgent: MultiAgentConfig.defaults().copyWith(enabled: false),
|
||||
),
|
||||
);
|
||||
|
||||
final controller = InstalledSkillE2EAppControllerInternal(
|
||||
store: store,
|
||||
sendGate: Completer<void>(),
|
||||
outputRelativePath: testCase.outputRelativePath,
|
||||
outputContent: testCase.outputContent,
|
||||
importedSkill: AssistantThreadSkillEntry(
|
||||
key: testCase.skillKey,
|
||||
label: testCase.skillKey,
|
||||
description: 'Installed skill under test',
|
||||
sourcePath: '${skillsRoot.path}/${testCase.skillKey}',
|
||||
sourceLabel: testCase.skillKey,
|
||||
),
|
||||
singleAgentSharedSkillScanRootOverrides: <String>[skillsRoot.path],
|
||||
);
|
||||
addTearDown(controller.dispose);
|
||||
await waitForConditionInternal(() => !controller.initializing);
|
||||
return controller;
|
||||
}
|
||||
|
||||
class CaptureSendAppControllerInternal extends AppController {
|
||||
CaptureSendAppControllerInternal({
|
||||
required SecureConfigStore store,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user