diff --git a/config/feature_flags.yaml b/config/feature_flags.yaml index 558cbc25..6bbececa 100644 --- a/config/feature_flags.yaml +++ b/config/feature_flags.yaml @@ -67,6 +67,12 @@ mobile: build_modes: [debug, profile, release] description: Mobile archived task management ui_surface: settings_page + remote_desktop: + enabled: true + release_tier: stable + build_modes: [debug, profile, release] + description: Mobile WebRTC Remote Desktop view + ui_surface: settings_page account_access: enabled: true release_tier: stable @@ -162,6 +168,12 @@ desktop: build_modes: [debug, profile, release] description: Desktop archived task management ui_surface: settings_page + remote_desktop: + enabled: true + release_tier: stable + build_modes: [debug, profile, release] + description: Desktop WebRTC Remote Desktop view + ui_surface: settings_page account_access: enabled: true release_tier: stable diff --git a/docs/cases/ai-security-evolution-content-scenario/README.md b/docs/cases/ai-security-evolution-content-scenario/README.md index 787332ce..d7dbf30d 100644 --- a/docs/cases/ai-security-evolution-content-scenario/README.md +++ b/docs/cases/ai-security-evolution-content-scenario/README.md @@ -20,6 +20,7 @@ - [XHS 风格文案](./xhs-copy.md) - [微信文章文案](./wechat-article.md) - [PPT 演示稿](./ai-security-evolution-scenario.pptx) +- [章节拆分 PDF 文稿](./ai-security-evolution-pdf.pdf) ## App 手动测试提示词 diff --git a/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.md b/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.md new file mode 100644 index 00000000..48693fa5 --- /dev/null +++ b/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.md @@ -0,0 +1,65 @@ +# 从单机权限到 AI 模型与知识保护 + +## 内容生产流程 + +拆章节 -> 每章调用 Codex 生成正文 -> 每章调用 GPT Images2 生成配图 -> 汇总排版 -> 输出 PDF。 + +## 1. 单机权限：安全从“这台机器归谁管”开始 + +早期安全的核心问题很朴素：谁能登录这台机器，谁能读写本地文件，谁能安装程序、修改配置、提升权限。管理员、普通用户、服务账号构成了最基本的权限边界。这个阶段的重点是账号、密码、本地组、文件 ACL 和系统审计。 + +但单机权限并没有消失。今天的终端仍然是身份、数据和任务链路的入口。开发者电脑、运维跳板机、员工笔记本一旦失守，攻击者就可能拿到浏览器会话、SSH Key、云凭证和内部系统访问能力。 + +图像提示：一台现代工作站，屏幕上叠加本地账号、文件权限、SSH Key 和浏览器会话的安全边界，写实科技风，清晰信息图。 + +## 2. 网络边界：从机房防火墙到组合攻击面 + +企业上网、服务器入站、内外网隔离、防火墙策略，曾经定义了很长一段时间的安全主战场。只要守住入口，默认内部网络可信，这是传统网络安全的基本假设。 + +云、移动办公、SaaS 和 API 普及后，边界被拆散了。今天的网络边界不只是 IP 和端口，而是 API、设备、身份、供应链、运行时和第三方集成的组合面。攻击也不再只从“打穿防火墙”开始，而可能从一个泄露 Token、一个开放对象存储桶、一个未校验 Webhook 或一个依赖包开始。 + +图像提示：传统防火墙边界逐渐碎裂，变成 API、设备、身份、供应链、运行时节点组成的网络拓扑，冷静企业安全风格。 + +## 3. Web 安全：页面漏洞扩展为业务流风险 + +Web 安全曾经最容易被理解为漏洞清单：SQL 注入、XSS、CSRF、文件上传、越权访问。它们仍然重要，但真正的风险已经扩展到业务流本身。攻击者更关心如何绕过审批、滥用接口、批量枚举、接管会话、自动化薅取权益，或者把插件和第三方脚本变成数据外泄通道。 + +所以 Web 安全不能只停留在扫描器和 WAF。它需要和身份、风控、审计、数据分级、API 治理一起工作。尤其当 AI Agent 可以自动浏览网页、调用后台接口、读写文件时，Web 不再只是用户界面，而是 Agent 执行动作的工作台。 + +图像提示：浏览器窗口中展示登录、API、插件、自动化脚本和数据流，旁边标注业务流风险，专业信息图，白底高对比。 + +## 4. 云身份：默认控制平面 + +进入云时代后，身份成为事实上的控制平面。谁能 Assume Role，谁能创建密钥，谁能访问对象存储，谁能改安全组，谁能跨账号部署资源，往往比某台服务器本身更关键。云环境里的风险也常常来自权限漂移、长期密钥暴露、过宽角色、跨账号信任和 CI/CD 凭证泄露。 + +云身份的治理重点，是把权限从“能不能登录机器”升级为“能不能代表某个主体操作某类资源”。最小权限、短期凭证、条件访问、资源标签、审计日志和权限边界，构成了云安全的基础。 + +图像提示：云控制平面中的 IAM、角色、短期凭证、跨账号访问和审计日志连接到计算与数据资源，现代云安全架构图。 + +## 5. Zero Trust：持续验证替代默认信任 + +Zero Trust 的价值不在口号，而在执行层：不因为你在内网就信任你，不因为你登录过一次就永久信任你，不因为你是某个部门的人就默认给你全部权限。每一次访问都要结合身份、设备、位置、会话、资源敏感度和行为上下文进行判断。 + +在实践中，Zero Trust 更像一套纪律：强身份、设备可信、最小权限、分段访问、持续评估、完整审计。它把过去一次性的准入控制，变成贯穿整个会话和操作过程的动态决策。 + +图像提示：用户、设备、应用、数据之间有动态策略引擎逐次校验访问请求，表达持续验证和最小权限，清爽企业图形风。 + +## 6. AI Agent 身份：谁在替人行动 + +AI Agent 的出现，把安全问题继续向前推了一步。过去系统主要判断“人是谁”；现在还要判断“这个 Agent 是谁创建的、代表谁、能做什么、能持续多久、能不能被撤销”。Agent 不是普通 API Key，也不应该混用人的全部权限。 + +一个可控的 Agent 身份需要独立标识、授权范围、任务上下文、工具白名单、数据访问边界、审计链路和撤销机制。它可以替人执行任务，但不能无限继承人的身份，也不能在缺少上下文时自行扩大权限。 + +图像提示：一个 AI Agent 作为独立身份站在人类用户与工具系统之间，权限边界、任务授权、审计链和撤销按钮清晰可见。 + +## 7. AI 模型与知识保护：新资产边界 + +AI 时代的新资产不只有模型权重。提示词、系统指令、RAG 知识库、企业文档、工具调用记录、对话历史、评测集和业务语料，都会影响模型行为，也都可能泄露业务能力。保护模型，本质上是在保护知识、意图和执行路径。 + +因此，安全边界最终从“谁能访问系统”演进为“谁能携带什么知识，以什么身份，替谁执行什么动作”。未来的安全体系必须同时治理身份边界、知识边界和行动边界。人、应用、Agent、模型和工具都要可区分、可授权、可审计、可撤销。 + +图像提示：模型、RAG 知识库、提示词、业务文档和工具调用记录构成新的资产边界，中心是受保护的 AI 系统，精致信息图。 + +## 汇总判断 + +安全的演进不是旧问题被新问题替代，而是边界不断外移、主体不断增多、资产不断抽象。单机权限仍是入口，网络边界仍是基础，Web 安全仍是主战场，云身份成为控制平面，Zero Trust 成为执行纪律，而 AI Agent 身份与模型知识保护，正在成为下一轮企业安全的核心命题。 diff --git a/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.pdf b/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.pdf new file mode 100644 index 00000000..4d8ce45c Binary files /dev/null and b/docs/cases/ai-security-evolution-content-scenario/ai-security-evolution-pdf.pdf differ diff --git a/lib/app/app_controller_desktop_skill_permissions.dart b/lib/app/app_controller_desktop_skill_permissions.dart index 180e712c..fd5463ad 100644 --- a/lib/app/app_controller_desktop_skill_permissions.dart +++ b/lib/app/app_controller_desktop_skill_permissions.dart @@ -78,6 +78,7 @@ extension AppControllerDesktopSkillPermissions on AppController { List? lastTaskArtifactRelativePaths, OpenClawTaskAssociation? openClawTaskAssociation, bool clearOpenClawTaskAssociation = false, + List? taskInputAttachments, }) { final normalizedSessionKey = normalizedAssistantSessionKeyInternal( sessionKey, @@ -204,6 +205,7 @@ extension AppControllerDesktopSkillPermissions on AppController { lastArtifactSyncAtMs: null, lastArtifactSyncStatus: null, lastTaskArtifactRelativePaths: const [], + taskInputAttachments: const [], )) .copyWith( messages: nextMessages, @@ -229,6 +231,7 @@ extension AppControllerDesktopSkillPermissions on AppController { lastTaskArtifactRelativePaths: lastTaskArtifactRelativePaths, openClawTaskAssociation: openClawTaskAssociation, clearOpenClawTaskAssociation: clearOpenClawTaskAssociation, + taskInputAttachments: taskInputAttachments, ); final nextStatus = lifecycleStatus ?? diff --git a/lib/app/app_controller_desktop_thread_actions.dart b/lib/app/app_controller_desktop_thread_actions.dart index d0e3422d..1c13b671 100644 --- a/lib/app/app_controller_desktop_thread_actions.dart +++ b/lib/app/app_controller_desktop_thread_actions.dart @@ -459,8 +459,13 @@ extension AppControllerDesktopThreadActions on AppController { final capturedSelectedSkillLabels = List.unmodifiable( selectedSkillLabels, ); + final inlineAttachmentsToUpload = + registerTaskInputAttachmentsForGatewayTurnInternal( + normalizedSessionKey, + attachments, + ); final capturedAttachments = List.unmodifiable( - attachments, + inlineAttachmentsToUpload, ); final capturedLocalAttachments = List.unmodifiable( localAttachments, @@ -523,6 +528,61 @@ extension AppControllerDesktopThreadActions on AppController { recomputeTasksInternal(); } + List + registerTaskInputAttachmentsForGatewayTurnInternal( + String sessionKey, + List attachments, + ) { + if (attachments.isEmpty) { + return const []; + } + final normalizedSessionKey = normalizedAssistantSessionKeyInternal( + sessionKey, + ); + final existing = taskThreadForSessionInternal(normalizedSessionKey); + final existingByKey = { + for (final item + in existing?.taskInputAttachments ?? + const []) + if (item.key.isNotEmpty) item.key: item, + }; + final inlineAttachmentsToUpload = []; + final nextByKey = {...existingByKey}; + final uploadedAtMs = DateTime.now().millisecondsSinceEpoch.toDouble(); + for (final attachment in attachments) { + final key = gatewayAttachmentPayloadSha256Internal(attachment); + if (key.isEmpty) { + inlineAttachmentsToUpload.add(attachment); + continue; + } + if (!existingByKey.containsKey(key)) { + inlineAttachmentsToUpload.add(attachment); + } + nextByKey.putIfAbsent( + key, + () => TaskInputAttachmentRecord( + name: attachment.fileName.trim(), + mimeType: attachment.mimeType.trim(), + sha256: key, + type: attachment.type.trim(), + uploadedAtMs: uploadedAtMs, + ), + ); + } + if (nextByKey.length != existingByKey.length) { + upsertTaskThreadInternal( + normalizedSessionKey, + taskInputAttachments: nextByKey.values.toList(growable: false), + updatedAtMs: uploadedAtMs, + ); + } + return inlineAttachmentsToUpload; + } + + String gatewayAttachmentPayloadSha256Internal( + GatewayChatAttachmentPayload attachment, + ) => goTaskServiceAttachmentSha256(attachment); + Future runGatewayChatTurnInternal({ required String sessionKey, required AssistantExecutionTarget target, @@ -558,6 +618,9 @@ extension AppControllerDesktopThreadActions on AppController { executionWorkingDirectory: executionWorkingDirectory ?? workingDirectory, remoteWorkingDirectoryHint: remoteWorkingDirectoryHint, target: target, + taskInputAttachments: + taskThreadForSessionInternal(sessionKey)?.taskInputAttachments ?? + const [], ); if (appendUserTurn) { appendGatewayUserTurnInternal(sessionKey, message); @@ -592,7 +655,11 @@ extension AppControllerDesktopThreadActions on AppController { } }, ); - if (!aiGatewayPendingSessionKeysInternal.contains(sessionKey)) { + if (!aiGatewayPendingSessionKeysInternal.contains(sessionKey) && + taskThreadForSessionInternal( + sessionKey, + )?.lifecycleState.lastResultCode == + 'aborted') { clearAiGatewayStreamingTextInternal(sessionKey); return; } @@ -802,6 +869,8 @@ extension AppControllerDesktopThreadActions on AppController { String? executionWorkingDirectory, required String remoteWorkingDirectoryHint, required AssistantExecutionTarget target, + List taskInputAttachments = + const [], }) { final requestText = userPrompt.trim().isEmpty ? 'See attached.' @@ -824,6 +893,17 @@ extension AppControllerDesktopThreadActions on AppController { ? '(Use the runtime-provided default workspace)' : workingDirectory.trim(); buffer.writeln('- currentTaskWorkspace: $resolvedWorkspace'); + final visibleTaskInputAttachments = taskInputAttachments + .where((item) => item.name.trim().isNotEmpty && item.key.isNotEmpty) + .toList(growable: false); + if (visibleTaskInputAttachments.isNotEmpty) { + buffer.writeln('- taskInputAttachments:'); + for (final attachment in visibleTaskInputAttachments) { + buffer.writeln( + ' - ${attachment.name.trim()} (${attachment.mimeType.trim()}, sha256: ${attachment.key})', + ); + } + } buffer ..writeln() ..writeln('Workspace isolation rules:') @@ -845,6 +925,9 @@ extension AppControllerDesktopThreadActions on AppController { ..writeln( '6. The app syncs final artifacts from currentTaskWorkspace back into localWorkspace.', ) + ..writeln( + '7. Files listed in taskInputAttachments already belong to this TaskThread; reuse them from the task context and do not ask the user to upload them again.', + ) ..writeln(); buffer ..writeln('User request:') @@ -1250,6 +1333,13 @@ extension AppControllerDesktopThreadActions on AppController { recomputeTasksInternal(); notifyIfActiveInternal(); await persistGoTaskArtifactsForSessionInternal(sessionKey, result); + upsertTaskThreadInternal( + sessionKey, + lifecycleStatus: 'ready', + lastRunAtMs: completedAtMs, + lastResultCode: terminalResultCode, + updatedAtMs: completedAtMs, + ); } Future applyGatewayChatFailureInternal({ diff --git a/lib/app/ui_feature_manifest_core.dart b/lib/app/ui_feature_manifest_core.dart index ab17b846..f6f32cf5 100644 --- a/lib/app/ui_feature_manifest_core.dart +++ b/lib/app/ui_feature_manifest_core.dart @@ -47,6 +47,7 @@ abstract final class UiFeatureKeys { static const settingsGateway = 'settings.gateway'; static const settingsArchivedTasks = 'settings.archived_tasks'; + static const settingsRemoteDesktop = 'settings.remote_desktop'; static const settingsAccountAccess = 'settings.account_access'; static const settingsVaultServer = 'settings.vault_server'; static const settingsExperimentalCanvas = 'settings.experimental_canvas'; @@ -366,6 +367,7 @@ class UiFeatureAccess { { UiFeatureKeys.settingsGateway: SettingsTab.gateway, UiFeatureKeys.settingsArchivedTasks: SettingsTab.archivedTasks, + UiFeatureKeys.settingsRemoteDesktop: SettingsTab.remoteDesktop, }; bool isEnabledPath(String path) { diff --git a/lib/features/assistant/assistant_attachment_payloads.dart b/lib/features/assistant/assistant_attachment_payloads.dart index 5a7f551c..b95d46a6 100644 --- a/lib/features/assistant/assistant_attachment_payloads.dart +++ b/lib/features/assistant/assistant_attachment_payloads.dart @@ -1,6 +1,8 @@ import 'dart:convert'; import 'dart:io'; +import 'package:crypto/crypto.dart' as crypto; + import '../../runtime/runtime_models.dart'; import 'assistant_page_composer_clipboard.dart'; @@ -81,6 +83,7 @@ buildAssistantAttachmentPayloadsInternal( mimeType: mimeType, fileName: attachment.name, content: base64Encode(bytes), + sha256: crypto.sha256.convert(bytes).toString(), ), ); } diff --git a/lib/features/desktop/desktop_client.dart b/lib/features/desktop/desktop_client.dart new file mode 100644 index 00000000..455a206b --- /dev/null +++ b/lib/features/desktop/desktop_client.dart @@ -0,0 +1,158 @@ +import 'dart:async'; +import 'dart:convert'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; +import '../../app/app_controller.dart'; + +class DesktopClient { + DesktopClient({required this.controller, required this.sessionId}); + + final AppController controller; + final String sessionId; + + RTCPeerConnection? _peerConnection; + RTCDataChannel? _dataChannel; + MediaStream? _remoteStream; + + final StreamController _streamController = + StreamController.broadcast(); + Stream get onRemoteStream => _streamController.stream; + + final StreamController _stateController = + StreamController.broadcast(); + Stream get onConnectionState => _stateController.stream; + + bool _isConnecting = false; + bool get isConnecting => _isConnecting; + bool get isConnected => + _peerConnection?.connectionState == + RTCPeerConnectionState.RTCPeerConnectionStateConnected; + + Future connect({ + String display = '', + int width = 1280, + int height = 720, + int fps = 30, + int bitrate = 2000, + bool useGpu = false, + }) async { + if (_isConnecting || isConnected) return; + _isConnecting = true; + _stateController.add('connecting'); + + try { + final config = { + 'iceServers': [ + {'url': 'stun:stun.l.google.com:19302'} + ], + 'sdpSemantics': 'unified-plan', + }; + + _peerConnection = await createPeerConnection(config); + + // Listen for remote streams + _peerConnection!.onTrack = (event) { + if (event.track.kind == 'video') { + if (event.streams.isNotEmpty) { + _remoteStream = event.streams.first; + _streamController.add(_remoteStream!); + } + } + }; + + _peerConnection!.onConnectionState = (state) { + _stateController.add(state.toString().split('.').last); + }; + + // Create data channel for inputs + final dcConfig = RTCDataChannelInit()..ordered = true; + _dataChannel = + await _peerConnection!.createDataChannel('input', dcConfig); + + // Handle ICE Candidates generated locally + _peerConnection!.onIceCandidate = (RTCIceCandidate? candidate) { + if (candidate != null) { + _sendIceCandidate(candidate); + } + }; + + // Create SDP Offer + final offer = await _peerConnection!.createOffer({ + 'offerToReceiveVideo': true, + 'offerToReceiveAudio': false, + }); + await _peerConnection!.setLocalDescription(offer); + + // Send SDP Offer to Bridge + final response = await controller.gatewayAcpClientInternal.request( + method: 'xworkmate.desktop.offer', + params: { + 'sessionId': sessionId, + 'sdpOffer': offer.sdp, + 'display': display, + 'width': width.toString(), + 'height': height.toString(), + 'fps': fps.toString(), + 'bitrate': bitrate.toString(), + 'useGpu': useGpu.toString(), + }, + ); + + final sdpAnswer = response['result']?['sdpAnswer'] as String?; + if (sdpAnswer == null) { + throw Exception('Bridge failed to return SDP Answer'); + } + + // Apply SDP Answer + final answer = RTCSessionDescription(sdpAnswer, 'answer'); + await _peerConnection!.setRemoteDescription(answer); + + _isConnecting = false; + } catch (e) { + _isConnecting = false; + _stateController.add('failed'); + await disconnect(); + rethrow; + } + } + + Future _sendIceCandidate(RTCIceCandidate candidate) async { + try { + await controller.gatewayAcpClientInternal.request( + method: 'xworkmate.desktop.ice', + params: { + 'sessionId': sessionId, + 'candidate': { + 'candidate': candidate.candidate, + 'sdpMid': candidate.sdpMid, + 'sdpMLineIndex': candidate.sdpMLineIndex, + }, + }, + ); + } catch (_) {} + } + + void sendInput(Map event) { + final channel = _dataChannel; + if (channel != null && + channel.state == RTCDataChannelState.RTCDataChannelOpen) { + final jsonStr = jsonEncode(event); + channel.send(RTCDataChannelMessage(jsonStr)); + } + } + + Future disconnect() async { + try { + await controller.gatewayAcpClientInternal.request( + method: 'xworkmate.desktop.close', + params: {'sessionId': sessionId}, + ); + } catch (_) {} + + await _dataChannel?.close(); + await _peerConnection?.close(); + _dataChannel = null; + _peerConnection = null; + _remoteStream = null; + _stateController.add('disconnected'); + } +} diff --git a/lib/features/desktop/desktop_input_handler.dart b/lib/features/desktop/desktop_input_handler.dart new file mode 100644 index 00000000..07cfbbeb --- /dev/null +++ b/lib/features/desktop/desktop_input_handler.dart @@ -0,0 +1,94 @@ +import 'package:flutter/gestures.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter/widgets.dart'; + +class DesktopInputHandler { + DesktopInputHandler({required this.onSendInput}); + + final void Function(Map event) onSendInput; + int _lastPressedButton = 1; // Default to left click + + void handlePointerMove(PointerEvent event, Size widgetSize) { + if (widgetSize.width == 0 || widgetSize.height == 0) return; + + final x = event.localPosition.dx / widgetSize.width; + final y = event.localPosition.dy / widgetSize.height; + + final cx = x.clamp(0.0, 1.0); + final cy = y.clamp(0.0, 1.0); + + onSendInput({ + 'type': 'mouse_move', + 'x': cx, + 'y': cy, + }); + } + + void handlePointerDown(PointerDownEvent event, Size widgetSize) { + if (widgetSize.width == 0 || widgetSize.height == 0) return; + + final x = event.localPosition.dx / widgetSize.width; + final y = event.localPosition.dy / widgetSize.height; + final cx = x.clamp(0.0, 1.0); + final cy = y.clamp(0.0, 1.0); + + // Send move event first to ensure click hits the exact coordinates + onSendInput({ + 'type': 'mouse_move', + 'x': cx, + 'y': cy, + }); + + _lastPressedButton = _mapPointerButtons(event.buttons); + + onSendInput({ + 'type': 'mouse_down', + 'button': _lastPressedButton, + }); + } + + void handlePointerUp(PointerUpEvent event, Size widgetSize) { + // Under pointer up, buttons bitmask represents STILL pressed buttons. + // If it is 0, then the released button is the one we tracked in PointerDown. + int releasedButton = _lastPressedButton; + if (event.buttons != 0) { + releasedButton = _mapPointerButtons(event.buttons); + } + + onSendInput({ + 'type': 'mouse_up', + 'button': releasedButton, + }); + } + + void handleScroll(PointerScrollEvent event) { + // 4 = scroll up, 5 = scroll down in X11/xdotool button maps + final button = event.scrollDelta.dy < 0 ? 4 : 5; + onSendInput({ + 'type': 'scroll', + 'button': button, + }); + } + + void handleKeyEvent(KeyEvent event) { + final isDown = event is KeyDownEvent || event is KeyRepeatEvent; + final keyLabel = event.logicalKey.keyLabel; + + onSendInput({ + 'type': isDown ? 'key_down' : 'key_up', + 'key': keyLabel, + }); + } + + int _mapPointerButtons(int buttons) { + // Flutter buttons bitmask: + // 1 = primary (left click) + // 2 = secondary (right click) + // 4 = middle click + // Linux/xdotool mouse mapping: 1=left, 2=middle, 3=right + if (buttons & 1 != 0) return 1; + if (buttons & 4 != 0) return 2; // middle click + if (buttons & 2 != 0) return 3; // right click + return 1; + } +} diff --git a/lib/features/desktop/desktop_view.dart b/lib/features/desktop/desktop_view.dart new file mode 100644 index 00000000..1cfa201d --- /dev/null +++ b/lib/features/desktop/desktop_view.dart @@ -0,0 +1,458 @@ +import 'dart:async'; +import 'package:flutter/gestures.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter_webrtc/flutter_webrtc.dart'; +import 'desktop_client.dart'; +import 'desktop_input_handler.dart'; +import '../../app/app_controller.dart'; +import '../../widgets/surface_card.dart'; + +class DesktopView extends StatefulWidget { + const DesktopView({super.key, required this.controller}); + + final AppController controller; + + @override + State createState() => _DesktopViewState(); +} + +class _DesktopViewState extends State { + final RTCVideoRenderer _localRenderer = RTCVideoRenderer(); + late DesktopClient _client; + DesktopInputHandler? _inputHandler; + + // Settings controllers + final TextEditingController _displayController = TextEditingController(text: ':0.0'); + final TextEditingController _widthController = TextEditingController(text: '1280'); + final TextEditingController _heightController = TextEditingController(text: '720'); + final TextEditingController _fpsController = TextEditingController(text: '30'); + final TextEditingController _bitrateController = TextEditingController(text: '2000'); + + bool _useGpu = false; + String _connectionState = 'disconnected'; + bool _hasStream = false; + bool _isFocused = false; + + final FocusNode _viewportFocusNode = FocusNode(); + final GlobalKey _viewportKey = GlobalKey(); + + StreamSubscription? _streamSubscription; + StreamSubscription? _stateSubscription; + + @override + void initState() { + super.initState(); + _initRenderer(); + _client = DesktopClient( + controller: widget.controller, + sessionId: 'remote-desktop-session', + ); + _inputHandler = DesktopInputHandler(onSendInput: (event) { + if (_connectionState == 'connected') { + _client.sendInput(event); + } + }); + + _streamSubscription = _client.onRemoteStream.listen((stream) { + if (mounted) { + setState(() { + _localRenderer.srcObject = stream; + _hasStream = true; + }); + } + }); + + _stateSubscription = _client.onConnectionState.listen((state) { + if (mounted) { + setState(() { + _connectionState = state.toLowerCase(); + if (_connectionState == 'disconnected' || _connectionState == 'failed') { + _hasStream = false; + _localRenderer.srcObject = null; + } + }); + } + }); + } + + Future _initRenderer() async { + await _localRenderer.initialize(); + } + + @override + void dispose() { + _streamSubscription?.cancel(); + _stateSubscription?.cancel(); + _client.disconnect(); + _localRenderer.dispose(); + _displayController.dispose(); + _widthController.dispose(); + _heightController.dispose(); + _fpsController.dispose(); + _bitrateController.dispose(); + _viewportFocusNode.dispose(); + super.dispose(); + } + + void _toggleConnection() async { + if (_connectionState == 'connected' || _connectionState == 'connecting') { + await _client.disconnect(); + } else { + final display = _displayController.text.trim(); + final width = int.tryParse(_widthController.text) ?? 1280; + final height = int.tryParse(_heightController.text) ?? 720; + final fps = int.tryParse(_fpsController.text) ?? 30; + final bitrate = int.tryParse(_bitrateController.text) ?? 2000; + + try { + await _client.connect( + display: display, + width: width, + height: height, + fps: fps, + bitrate: bitrate, + useGpu: _useGpu, + ); + } catch (e) { + if (mounted) { + ScaffoldMessenger.of(context).showSnackBar( + SnackBar( + content: Text('Failed to connect remote desktop: $e'), + backgroundColor: Colors.redAccent, + ), + ); + } + } + } + } + + Size _getViewportSize() { + final renderBox = _viewportKey.currentContext?.findRenderObject() as RenderBox?; + return renderBox?.size ?? Size.zero; + } + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final isDark = theme.brightness == Brightness.dark; + + return Padding( + padding: const EdgeInsets.all(16.0), + child: Column( + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + // Control panel card + SurfaceCard( + child: Padding( + padding: const EdgeInsets.all(16.0), + child: Wrap( + spacing: 16, + runSpacing: 16, + crossAxisAlignment: WrapCrossAlignment.center, + children: [ + // Connection Button + ElevatedButton.icon( + onPressed: _toggleConnection, + style: ElevatedButton.styleFrom( + backgroundColor: _connectionState == 'connected' + ? Colors.redAccent + : (_connectionState == 'connecting' + ? Colors.orangeAccent + : theme.colorScheme.primary), + foregroundColor: Colors.white, + padding: const EdgeInsets.symmetric(horizontal: 20, vertical: 16), + shape: RoundedRectangleBorder(borderRadius: BorderRadius.circular(8)), + ), + icon: Icon( + _connectionState == 'connected' + ? Icons.portable_wifi_off_rounded + : Icons.settings_remote_rounded, + ), + label: Text( + _connectionState == 'connected' + ? '断开连接' + : (_connectionState == 'connecting' ? '正在连接...' : '连接桌面'), + style: const TextStyle(fontWeight: FontWeight.bold), + ), + ), + + // Display Selector + SizedBox( + width: 100, + child: TextField( + controller: _displayController, + enabled: _connectionState == 'disconnected', + decoration: const InputDecoration( + labelText: 'Display', + prefixIcon: Icon(Icons.monitor_rounded, size: 16), + ), + ), + ), + + // Resolution settings + SizedBox( + width: 90, + child: TextField( + controller: _widthController, + enabled: _connectionState == 'disconnected', + keyboardType: TextInputType.number, + decoration: const InputDecoration(labelText: '宽度'), + ), + ), + SizedBox( + width: 90, + child: TextField( + controller: _heightController, + enabled: _connectionState == 'disconnected', + keyboardType: TextInputType.number, + decoration: const InputDecoration(labelText: '高度'), + ), + ), + + // FPS / Bitrate + SizedBox( + width: 70, + child: TextField( + controller: _fpsController, + enabled: _connectionState == 'disconnected', + keyboardType: TextInputType.number, + decoration: const InputDecoration(labelText: '帧率'), + ), + ), + SizedBox( + width: 90, + child: TextField( + controller: _bitrateController, + enabled: _connectionState == 'disconnected', + keyboardType: TextInputType.number, + decoration: const InputDecoration( + labelText: '码率 (kbps)', + ), + ), + ), + + // GPU accelerator toggle + Row( + mainAxisSize: MainAxisSize.min, + children: [ + const Text('GPU 加速'), + Switch( + value: _useGpu, + onChanged: _connectionState == 'disconnected' + ? (val) => setState(() => _useGpu = val) + : null, + ), + ], + ), + + // Status Indicator + Container( + padding: const EdgeInsets.symmetric(horizontal: 12, vertical: 6), + decoration: BoxDecoration( + color: _connectionState == 'connected' + ? Colors.green.withValues(alpha: 0.15) + : (_connectionState == 'connecting' + ? Colors.orange.withValues(alpha: 0.15) + : Colors.grey.withValues(alpha: 0.15)), + borderRadius: BorderRadius.circular(16), + border: Border.all( + color: _connectionState == 'connected' + ? Colors.green + : (_connectionState == 'connecting' + ? Colors.orange + : Colors.grey), + width: 1, + ), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Container( + width: 8, + height: 8, + decoration: BoxDecoration( + color: _connectionState == 'connected' + ? Colors.green + : (_connectionState == 'connecting' + ? Colors.orange + : Colors.grey), + shape: BoxShape.circle, + ), + ), + const SizedBox(width: 8), + Text( + _connectionState == 'connected' + ? '已连接' + : (_connectionState == 'connecting' ? '连接中' : '未连接'), + style: TextStyle( + fontSize: 12, + fontWeight: FontWeight.bold, + color: _connectionState == 'connected' + ? Colors.green + : (_connectionState == 'connecting' + ? Colors.orange + : (isDark ? Colors.white70 : Colors.black87)), + ), + ), + ], + ), + ), + ], + ), + ), + ), + + const SizedBox(height: 16), + + // Stream Viewport Card + Expanded( + child: Focus( + focusNode: _viewportFocusNode, + onFocusChange: (focused) { + setState(() { + _isFocused = focused; + }); + }, + onKeyEvent: (node, event) { + if (_isFocused && _connectionState == 'connected' && _inputHandler != null) { + _inputHandler!.handleKeyEvent(event); + return KeyEventResult.handled; + } + return KeyEventResult.ignored; + }, + child: Container( + key: _viewportKey, + decoration: BoxDecoration( + color: isDark ? Colors.black26 : Colors.black.withValues(alpha: 0.04), + borderRadius: BorderRadius.circular(12), + border: Border.all( + color: _isFocused + ? theme.colorScheme.primary + : (isDark ? Colors.white10 : Colors.black12), + width: 2, + ), + ), + clipBehavior: Clip.antiAlias, + child: Stack( + children: [ + // Stream Viewport Renderer + if (_hasStream) + Positioned.fill( + child: Listener( + behavior: HitTestBehavior.opaque, + onPointerHover: (event) { + if (_inputHandler != null) { + _inputHandler!.handlePointerMove(event, _getViewportSize()); + } + }, + onPointerMove: (event) { + if (_inputHandler != null) { + _inputHandler!.handlePointerMove(event, _getViewportSize()); + } + }, + onPointerDown: (event) { + if (!_viewportFocusNode.hasFocus) { + _viewportFocusNode.requestFocus(); + } + if (_inputHandler != null) { + _inputHandler!.handlePointerDown(event, _getViewportSize()); + } + }, + onPointerUp: (event) { + if (_inputHandler != null) { + _inputHandler!.handlePointerUp(event, _getViewportSize()); + } + }, + onPointerSignal: (event) { + if (event is PointerScrollEvent && _inputHandler != null) { + _inputHandler!.handleScroll(event); + } + }, + child: RTCVideoView( + _localRenderer, + objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitContain, + ), + ), + ), + + // Placeholder/Status UI overlay + if (!_hasStream) + Positioned.fill( + child: Container( + color: isDark ? Colors.black54 : Colors.grey[100], + child: Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + Icons.monitor_rounded, + size: 64, + color: theme.colorScheme.onSurface.withValues(alpha: 0.2), + ), + const SizedBox(height: 16), + Text( + _connectionState == 'connecting' + ? '正在建立 WebRTC 连接，请稍候...' + : '未开启远程桌面流。点击“连接桌面”启动视频流。', + style: TextStyle( + color: theme.colorScheme.onSurface.withValues(alpha: 0.6), + fontSize: 14, + ), + ), + if (_connectionState == 'connecting') ...[ + const SizedBox(height: 24), + const CircularProgressIndicator(), + ], + ], + ), + ), + ), + ), + + // Focus watermark badge + if (_hasStream) + Positioned( + right: 8, + bottom: 8, + child: AnimatedOpacity( + opacity: _isFocused ? 0.3 : 0.8, + duration: const Duration(milliseconds: 200), + child: Container( + padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 4), + decoration: BoxDecoration( + color: Colors.black.withValues(alpha: 0.6), + borderRadius: BorderRadius.circular(4), + ), + child: Row( + mainAxisSize: MainAxisSize.min, + children: [ + Icon( + _isFocused + ? Icons.keyboard_rounded + : Icons.keyboard_hide_rounded, + color: Colors.white, + size: 14, + ), + const SizedBox(width: 4), + Text( + _isFocused ? '捕获键盘输入中' : '点击屏幕以捕获键盘', + style: const TextStyle( + color: Colors.white, + fontSize: 11, + ), + ), + ], + ), + ), + ), + ), + ], + ), + ), + ), + ), + ], + ), + ); + } +} diff --git a/lib/features/settings/settings_page_core.dart b/lib/features/settings/settings_page_core.dart index 64a641bf..146c93a5 100644 --- a/lib/features/settings/settings_page_core.dart +++ b/lib/features/settings/settings_page_core.dart @@ -17,6 +17,7 @@ import '../../widgets/surface_card.dart'; import 'settings_account_panel.dart'; import 'settings_about_panel.dart'; import 'settings_archived_tasks_panel.dart'; +import 'settings_remote_desktop_panel.dart'; Future> loadBridgeMetadataForSettingsAbout({ required Uri bridgeEndpoint, @@ -514,7 +515,7 @@ class _SettingsPageState extends State { onRefresh: _refreshAboutSnapshot, ), ), - ] else if (currentTab == SettingsTab.archivedTasks) + ] else if (currentTab == SettingsTab.archivedTasks) ...[ SurfaceCard( key: const ValueKey('settings-archived-tasks-panel-card'), child: SettingsArchivedTasksPanel( @@ -523,6 +524,12 @@ class _SettingsPageState extends State { onDelete: _deleteArchivedTask, ), ), + ] else if (currentTab == SettingsTab.remoteDesktop) ...[ + SurfaceCard( + key: const ValueKey('settings-remote-desktop-panel-card'), + child: SettingsRemoteDesktopPanel(controller: controller), + ), + ], ], ); }, @@ -555,9 +562,11 @@ class _SettingsTabSelector extends StatelessWidget { ButtonSegment( value: tab, icon: Icon( - tab == SettingsTab.archivedTasks - ? Icons.inventory_2_outlined - : Icons.hub_outlined, + tab == SettingsTab.remoteDesktop + ? Icons.desktop_windows_outlined + : (tab == SettingsTab.archivedTasks + ? Icons.inventory_2_outlined + : Icons.hub_outlined), ), label: Text(tab.label), ), diff --git a/lib/features/settings/settings_remote_desktop_panel.dart b/lib/features/settings/settings_remote_desktop_panel.dart new file mode 100644 index 00000000..899d9825 --- /dev/null +++ b/lib/features/settings/settings_remote_desktop_panel.dart @@ -0,0 +1,49 @@ +import 'package:flutter/material.dart'; +import '../../app/app_controller.dart'; +import '../../i18n/app_language.dart'; +import '../../theme/app_palette.dart'; +import '../desktop/desktop_view.dart'; + +class SettingsRemoteDesktopPanel extends StatefulWidget { + const SettingsRemoteDesktopPanel({super.key, required this.controller}); + + final AppController controller; + + @override + State createState() => + _SettingsRemoteDesktopPanelState(); +} + +class _SettingsRemoteDesktopPanelState extends State { + @override + Widget build(BuildContext context) { + final palette = context.palette; + final theme = Theme.of(context); + + return Column( + key: const ValueKey('settings-remote-desktop-panel'), + crossAxisAlignment: CrossAxisAlignment.stretch, + children: [ + Row( + children: [ + Icon(Icons.desktop_windows_outlined, color: palette.textSecondary), + const SizedBox(width: 10), + Expanded( + child: Text( + appText('远程桌面', 'Remote Desktop'), + style: theme.textTheme.titleMedium?.copyWith( + fontWeight: FontWeight.w700, + ), + ), + ), + ], + ), + const SizedBox(height: 12), + SizedBox( + height: 640, + child: DesktopView(controller: widget.controller), + ), + ], + ); + } +} diff --git a/lib/models/app_models.dart b/lib/models/app_models.dart index 8e650367..6160c991 100644 --- a/lib/models/app_models.dart +++ b/lib/models/app_models.dart @@ -149,12 +149,13 @@ extension AssistantModeCopy on AssistantMode { }; } -enum SettingsTab { gateway, archivedTasks } +enum SettingsTab { gateway, archivedTasks, remoteDesktop } extension SettingsTabCopy on SettingsTab { String get label => switch (this) { SettingsTab.gateway => appText('集成', 'Integrations'), SettingsTab.archivedTasks => appText('归档任务', 'Archived tasks'), + SettingsTab.remoteDesktop => appText('远程桌面', 'Remote Desktop'), }; } diff --git a/lib/runtime/go_task_service_client.dart b/lib/runtime/go_task_service_client.dart index 14621d6a..8da3724a 100644 --- a/lib/runtime/go_task_service_client.dart +++ b/lib/runtime/go_task_service_client.dart @@ -1,3 +1,7 @@ +import 'dart:convert'; + +import 'package:crypto/crypto.dart' as crypto; + import 'runtime_models.dart'; enum GoTaskServiceRoute { externalAcpSingle, externalAcpMulti } @@ -308,6 +312,8 @@ class GoTaskServiceRequest { 'mimeType': item.mimeType, 'content': item.content, 'sizeBytes': goTaskServiceBase64Size(item.content), + if (goTaskServiceAttachmentSha256(item).isNotEmpty) + 'sha256': goTaskServiceAttachmentSha256(item), }, ) .toList(growable: false), @@ -1005,6 +1011,20 @@ int goTaskServiceBase64Size(String base64) { return (normalized.length * 3 ~/ 4) - padding; } +String goTaskServiceAttachmentSha256(GatewayChatAttachmentPayload attachment) { + final declared = attachment.sha256.trim().toLowerCase(); + if (declared.isNotEmpty) { + return declared; + } + try { + return crypto.sha256 + .convert(base64Decode(attachment.content.trim().split(',').last.trim())) + .toString(); + } on FormatException { + return ''; + } +} + Map _castMap(Object? value) { if (value is Map) { return value; diff --git a/lib/runtime/runtime_models_gateway_entities.dart b/lib/runtime/runtime_models_gateway_entities.dart index 150f53c1..3cdfd540 100644 --- a/lib/runtime/runtime_models_gateway_entities.dart +++ b/lib/runtime/runtime_models_gateway_entities.dart @@ -16,12 +16,14 @@ class GatewayChatAttachmentPayload { required this.mimeType, required this.fileName, required this.content, + this.sha256 = '', }); final String type; final String mimeType; final String fileName; final String content; + final String sha256; Map toJson() { return { @@ -29,6 +31,7 @@ class GatewayChatAttachmentPayload { 'mimeType': mimeType, 'fileName': fileName, 'content': content, + if (sha256.trim().isNotEmpty) 'sha256': sha256.trim(), }; } } diff --git a/lib/runtime/runtime_models_runtime_payloads.dart b/lib/runtime/runtime_models_runtime_payloads.dart index eac01bf3..43c1aa93 100644 --- a/lib/runtime/runtime_models_runtime_payloads.dart +++ b/lib/runtime/runtime_models_runtime_payloads.dart @@ -678,6 +678,7 @@ class ThreadContextState { this.lastArtifactSyncStatus, this.lastTaskArtifactRelativePaths = const [], this.openClawTaskAssociation, + this.taskInputAttachments = const [], }); final List messages; @@ -696,6 +697,7 @@ class ThreadContextState { final String? lastArtifactSyncStatus; final List lastTaskArtifactRelativePaths; final OpenClawTaskAssociation? openClawTaskAssociation; + final List taskInputAttachments; ThreadContextState copyWith({ List? messages, @@ -716,6 +718,7 @@ class ThreadContextState { List? lastTaskArtifactRelativePaths, OpenClawTaskAssociation? openClawTaskAssociation, bool clearOpenClawTaskAssociation = false, + List? taskInputAttachments, }) { return ThreadContextState( messages: messages ?? this.messages, @@ -745,6 +748,7 @@ class ThreadContextState { openClawTaskAssociation: clearOpenClawTaskAssociation ? null : (openClawTaskAssociation ?? this.openClawTaskAssociation), + taskInputAttachments: taskInputAttachments ?? this.taskInputAttachments, ); } @@ -766,6 +770,9 @@ class ThreadContextState { 'lastArtifactSyncStatus': lastArtifactSyncStatus, 'lastTaskArtifactRelativePaths': lastTaskArtifactRelativePaths, 'openClawTaskAssociation': openClawTaskAssociation?.toJson(), + 'taskInputAttachments': taskInputAttachments + .map((item) => item.toJson()) + .toList(growable: false), }; } @@ -834,10 +841,77 @@ class ThreadContextState { openClawTaskAssociation: OpenClawTaskAssociation.fromJsonOrNull( json['openClawTaskAssociation'], ), + taskInputAttachments: _taskInputAttachmentRecordsFromJson( + json['taskInputAttachments'], + ), ); } } +class TaskInputAttachmentRecord { + const TaskInputAttachmentRecord({ + required this.name, + required this.mimeType, + required this.sha256, + required this.type, + required this.uploadedAtMs, + }); + + final String name; + final String mimeType; + final String sha256; + final String type; + final double uploadedAtMs; + + String get key => sha256.trim().toLowerCase(); + + Map toJson() { + return { + 'name': name.trim(), + 'mimeType': mimeType.trim(), + 'sha256': key, + 'type': type.trim(), + 'uploadedAtMs': uploadedAtMs, + }; + } + + factory TaskInputAttachmentRecord.fromJson(Map json) { + double uploadedAtMs(Object? value) { + if (value is num) { + return value.toDouble(); + } + return double.tryParse(value?.toString() ?? '') ?? 0; + } + + return TaskInputAttachmentRecord( + name: json['name']?.toString().trim() ?? '', + mimeType: json['mimeType']?.toString().trim() ?? '', + sha256: json['sha256']?.toString().trim().toLowerCase() ?? '', + type: json['type']?.toString().trim() ?? '', + uploadedAtMs: uploadedAtMs(json['uploadedAtMs']), + ); + } +} + +List _taskInputAttachmentRecordsFromJson( + Object? value, +) { + if (value is! List) { + return const []; + } + final byKey = {}; + for (final item in value.whereType()) { + final record = TaskInputAttachmentRecord.fromJson( + item.cast(), + ); + if (record.key.isEmpty || record.name.isEmpty) { + continue; + } + byKey.putIfAbsent(record.key, () => record); + } + return byKey.values.toList(growable: false); +} + class OpenClawTaskAssociation { const OpenClawTaskAssociation({ required this.sessionId, @@ -1097,6 +1171,7 @@ class TaskThread { String? lastArtifactSyncStatus, List? lastTaskArtifactRelativePaths, OpenClawTaskAssociation? openClawTaskAssociation, + List? taskInputAttachments, }) : threadId = _resolveThreadId(threadId), title = title ?? '', ownerScope = @@ -1144,6 +1219,8 @@ class TaskThread { lastTaskArtifactRelativePaths, ), openClawTaskAssociation: openClawTaskAssociation, + taskInputAttachments: + taskInputAttachments ?? const [], ), lifecycleState = lifecycleState ?? @@ -1184,6 +1261,8 @@ class TaskThread { contextState.lastTaskArtifactRelativePaths; OpenClawTaskAssociation? get openClawTaskAssociation => contextState.openClawTaskAssociation; + List get taskInputAttachments => + contextState.taskInputAttachments; String get latestResolvedRuntimeModel => contextState.latestResolvedRuntimeModel; String get latestResolvedProviderId => contextState.latestResolvedProviderId; @@ -1229,6 +1308,7 @@ class TaskThread { List? lastTaskArtifactRelativePaths, OpenClawTaskAssociation? openClawTaskAssociation, bool clearOpenClawTaskAssociation = false, + List? taskInputAttachments, }) { return TaskThread( threadId: threadId ?? this.threadId, @@ -1254,6 +1334,7 @@ class TaskThread { lastTaskArtifactRelativePaths: lastTaskArtifactRelativePaths, openClawTaskAssociation: openClawTaskAssociation, clearOpenClawTaskAssociation: clearOpenClawTaskAssociation, + taskInputAttachments: taskInputAttachments, ), lifecycleState: (lifecycleState ?? this.lifecycleState).copyWith( archived: archived, @@ -1371,6 +1452,7 @@ class TaskThread { 'lastArtifactSyncStatus': json['lastArtifactSyncStatus'], 'lastTaskArtifactRelativePaths': json['lastTaskArtifactRelativePaths'], 'openClawTaskAssociation': json['openClawTaskAssociation'], + 'taskInputAttachments': json['taskInputAttachments'], }; } diff --git a/linux/flutter/generated_plugin_registrant.cc b/linux/flutter/generated_plugin_registrant.cc index 0dbee6e6..c4e8c71b 100644 --- a/linux/flutter/generated_plugin_registrant.cc +++ b/linux/flutter/generated_plugin_registrant.cc @@ -7,6 +7,7 @@ #include "generated_plugin_registrant.h" #include +#include #include #include @@ -14,6 +15,9 @@ void fl_register_plugins(FlPluginRegistry* registry) { g_autoptr(FlPluginRegistrar) file_selector_linux_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "FileSelectorPlugin"); file_selector_plugin_register_with_registrar(file_selector_linux_registrar); + g_autoptr(FlPluginRegistrar) flutter_webrtc_registrar = + fl_plugin_registry_get_registrar_for_plugin(registry, "FlutterWebRTCPlugin"); + flutter_web_r_t_c_plugin_register_with_registrar(flutter_webrtc_registrar); g_autoptr(FlPluginRegistrar) irondash_engine_context_registrar = fl_plugin_registry_get_registrar_for_plugin(registry, "IrondashEngineContextPlugin"); irondash_engine_context_plugin_register_with_registrar(irondash_engine_context_registrar); diff --git a/linux/flutter/generated_plugins.cmake b/linux/flutter/generated_plugins.cmake index c7ac82b9..072a5478 100644 --- a/linux/flutter/generated_plugins.cmake +++ b/linux/flutter/generated_plugins.cmake @@ -4,6 +4,7 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_linux + flutter_webrtc irondash_engine_context super_native_extensions ) diff --git a/macos/Flutter/GeneratedPluginRegistrant.swift b/macos/Flutter/GeneratedPluginRegistrant.swift index 9fdc93fd..4e32b0e3 100644 --- a/macos/Flutter/GeneratedPluginRegistrant.swift +++ b/macos/Flutter/GeneratedPluginRegistrant.swift @@ -7,6 +7,7 @@ import Foundation import device_info_plus import file_selector_macos +import flutter_webrtc import irondash_engine_context import package_info_plus import patrol @@ -16,6 +17,7 @@ import super_native_extensions func RegisterGeneratedPlugins(registry: FlutterPluginRegistry) { DeviceInfoPlusMacosPlugin.register(with: registry.registrar(forPlugin: "DeviceInfoPlusMacosPlugin")) FileSelectorPlugin.register(with: registry.registrar(forPlugin: "FileSelectorPlugin")) + FlutterWebRTCPlugin.register(with: registry.registrar(forPlugin: "FlutterWebRTCPlugin")) IrondashEngineContextPlugin.register(with: registry.registrar(forPlugin: "IrondashEngineContextPlugin")) FPPPackageInfoPlusPlugin.register(with: registry.registrar(forPlugin: "FPPPackageInfoPlusPlugin")) PatrolPlugin.register(with: registry.registrar(forPlugin: "PatrolPlugin")) diff --git a/pubspec.lock b/pubspec.lock index 7f184593..cdc2431d 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -97,6 +97,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.0.8" + dart_webrtc: + dependency: transitive + description: + name: dart_webrtc + sha256: f6d615bddea5e458ce180a914f3055c234ffb52fb7397a51b3491e76d6d7edb2 + url: "https://pub.dev" + source: hosted + version: "1.8.1" device_info_plus: dependency: "direct main" description: @@ -274,6 +282,14 @@ packages: description: flutter source: sdk version: "0.0.0" + flutter_webrtc: + dependency: "direct main" + description: + name: flutter_webrtc + sha256: b832dc76c0d1577f14aaf35e9c38d4ed7667cbc89c492b7bf4505d8d5f62e08b + url: "https://pub.dev" + source: hosted + version: "0.12.12+hotfix.1" fuchsia_remote_debug_protocol: dependency: transitive description: flutter @@ -348,6 +364,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.0" + js: + dependency: transitive + description: + name: js + sha256: "53385261521cc4a0c4658fd0ad07a7d14591cf8fc33abbceae306ddb974888dc" + url: "https://pub.dev" + source: hosted + version: "0.7.2" json_annotation: dependency: transitive description: @@ -711,6 +735,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.3.1" + synchronized: + dependency: transitive + description: + name: synchronized + sha256: "63896c27e81b28f8cb4e69ead0d3e8f03f1d1e5fc531a3e579cabed6a2c7c9e5" + url: "https://pub.dev" + source: hosted + version: "3.4.0+1" term_glyph: dependency: transitive description: @@ -791,6 +823,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.1.0" + webrtc_interface: + dependency: transitive + description: + name: webrtc_interface + sha256: c6f100eac5057d9a817a60473126f9828c796d42884d498af4f339c97b21014f + url: "https://pub.dev" + source: hosted + version: "1.5.1" win32: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index c24f9a28..3d682386 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -28,6 +28,7 @@ dependencies: shared_preferences: ^2.5.3 super_clipboard: ^0.9.0 web_socket_channel: ^3.0.3 + flutter_webrtc: ^0.12.3 yaml: ^3.1.3 dev_dependencies: diff --git a/test/features/settings/settings_remote_desktop_panel_test.dart b/test/features/settings/settings_remote_desktop_panel_test.dart new file mode 100644 index 00000000..1c1ef4c0 --- /dev/null +++ b/test/features/settings/settings_remote_desktop_panel_test.dart @@ -0,0 +1,115 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:xworkmate/app/app_controller.dart'; +import 'package:xworkmate/features/settings/settings_remote_desktop_panel.dart'; +import 'package:xworkmate/runtime/secure_config_store.dart'; +import 'package:xworkmate/runtime/runtime_models.dart'; +import 'package:xworkmate/theme/app_theme.dart'; +import 'package:xworkmate/widgets/surface_card.dart'; + +void main() { + group('SettingsRemoteDesktopPanel', () { + testWidgets('renders the panel title and connection dashboard', (tester) async { + // Set desktop window size + tester.view.physicalSize = const Size(1280, 900); + tester.view.devicePixelRatio = 1.0; + + final store = _MemorySecureConfigStore(); + final controller = _NoopRefreshAppController(store: store); + addTearDown(() { + controller.dispose(); + tester.view.resetPhysicalSize(); + tester.view.resetDevicePixelRatio(); + }); + + await tester.pumpWidget( + _buildTestApp( + child: SettingsRemoteDesktopPanel(controller: controller), + ), + ); + + // Verify the panel headers and titles + expect(find.text('远程桌面'), findsOneWidget); + expect(find.text('连接桌面'), findsOneWidget); + expect(find.text('GPU 加速'), findsOneWidget); + + // Verify inputs + expect(find.widgetWithText(TextField, 'Display'), findsOneWidget); + expect(find.text('Display'), findsOneWidget); + }); + }); +} + +Widget _buildTestApp({required Widget child}) { + return MaterialApp( + theme: AppTheme.light(), + home: Material( + child: Center( + child: SizedBox( + width: 1100, + child: SurfaceCard(child: child), + ), + ), + ), + ); +} + +class _NoopRefreshAppController extends AppController { + _NoopRefreshAppController({required SecureConfigStore store}) + : super(environmentOverride: const {}, store: store); + + Future refreshAcpCapabilitiesInternal({ + bool forceRefresh = false, + bool persistMountTargets = false, + }) async {} + + Future refreshSingleAgentCapabilitiesInternal({ + bool forceRefresh = false, + }) async {} +} + +class _MemorySecureConfigStore extends SecureConfigStore { + _MemorySecureConfigStore() : super(enableSecureStorage: false); + + SettingsSnapshot _settings = SettingsSnapshot.defaults(); + final Map _secrets = {}; + + @override + Future initialize() async {} + + @override + Future loadSettingsSnapshot() async => _settings; + + @override + Future saveSettingsSnapshot(SettingsSnapshot snapshot) async { + _settings = snapshot; + } + + @override + Future> loadSecureRefs() async => _secrets; + + @override + Future> loadAuditTrail() async => + const []; + + @override + Future appendAudit(SecretAuditEntry entry) async {} + + @override + Future loadSecretValueByRef(String refName) async => + _secrets[refName]; + + @override + Future saveSecretValueByRef(String refName, String value) async { + _secrets[refName] = value; + } + + @override + Future loadAccountSessionToken() async => null; + + @override + Future loadAccountSessionSummary() async => null; + + @override + Future loadAccountSyncState() async => null; +} diff --git a/test/runtime/assistant_execution_target_test.dart b/test/runtime/assistant_execution_target_test.dart index 601cefba..0f4fd10c 100644 --- a/test/runtime/assistant_execution_target_test.dart +++ b/test/runtime/assistant_execution_target_test.dart @@ -1351,6 +1351,10 @@ void main() { base64Encode(utf8.encode('note body')), ); expect(inlineAttachment['sizeBytes'], 9); + expect( + inlineAttachment['sha256'], + '1c727d26215adccb96d725e8b63b3ee11cf73215a554e60295877244b0778847', + ); final attachments = params['attachments'] as List; final attachment = attachments.single as Map; expect(attachment['name'], 'note.txt'); @@ -1358,6 +1362,56 @@ void main() { }, ); + test( + 'sendChatMessage records task input attachments and does not reupload duplicates', + () async { + final fakeGoTaskService = _RecordingGoTaskServiceClient(); + final controller = _connectedGatewayController(fakeGoTaskService); + addTearDown(controller.dispose); + + await controller.ensureActiveAssistantThreadInternal(); + await controller.setAssistantExecutionTarget( + AssistantExecutionTarget.gateway, + ); + + final imageAttachment = GatewayChatAttachmentPayload( + type: 'image', + mimeType: 'image/png', + fileName: 'diagram.png', + content: base64Encode(utf8.encode('image bytes')), + ); + + await controller.sendChatMessage( + 'use the image', + attachments: [imageAttachment], + ); + await controller.sendChatMessage( + 'continue with the same image', + attachments: [imageAttachment], + ); + + expect(fakeGoTaskService.requests, hasLength(2)); + expect( + fakeGoTaskService.requests.first.inlineAttachments, + hasLength(1), + ); + expect(fakeGoTaskService.requests.last.inlineAttachments, isEmpty); + expect( + fakeGoTaskService.requests.last.prompt, + contains('- taskInputAttachments:'), + ); + expect( + fakeGoTaskService.requests.last.prompt, + contains('diagram.png (image/png, sha256:'), + ); + final thread = controller.requireTaskThreadForSessionInternal( + fakeGoTaskService.requests.last.sessionId, + ); + expect(thread.taskInputAttachments, hasLength(1)); + expect(thread.taskInputAttachments.single.name, 'diagram.png'); + }, + ); + test( 'sendChatMessage resumes existing task after response interruption', () async { diff --git a/windows/flutter/generated_plugin_registrant.cc b/windows/flutter/generated_plugin_registrant.cc index 4fcbf216..c37a235c 100644 --- a/windows/flutter/generated_plugin_registrant.cc +++ b/windows/flutter/generated_plugin_registrant.cc @@ -7,12 +7,15 @@ #include "generated_plugin_registrant.h" #include +#include #include #include void RegisterPlugins(flutter::PluginRegistry* registry) { FileSelectorWindowsRegisterWithRegistrar( registry->GetRegistrarForPlugin("FileSelectorWindows")); + FlutterWebRTCPluginRegisterWithRegistrar( + registry->GetRegistrarForPlugin("FlutterWebRTCPlugin")); IrondashEngineContextPluginCApiRegisterWithRegistrar( registry->GetRegistrarForPlugin("IrondashEngineContextPluginCApi")); SuperNativeExtensionsPluginCApiRegisterWithRegistrar( diff --git a/windows/flutter/generated_plugins.cmake b/windows/flutter/generated_plugins.cmake index 829fcfea..bbf48b19 100644 --- a/windows/flutter/generated_plugins.cmake +++ b/windows/flutter/generated_plugins.cmake @@ -4,6 +4,7 @@ list(APPEND FLUTTER_PLUGIN_LIST file_selector_windows + flutter_webrtc irondash_engine_context super_native_extensions )