feat: enhance CDP bridge management and fix JS falsy returns

- background.js: Add management API, auto-return for JS, refactor new tab capture
- simphtml.py: Fix falsy return value bug, flatten return structure
- agent_loop.py: Simplify UI icons
- manifest.json: Add management permission
- tools_schema: Optimize prompts for web tools
- tmwebdriver_sop.md: Update documentation
This commit is contained in:
Liang Jiaqing
2026-04-08 18:28:16 +08:00
parent 813eb29692
commit 1c561db0b9
7 changed files with 86 additions and 108 deletions

View File

@@ -23,8 +23,7 @@ class BaseHandler:
ret = yield from try_call_generator(getattr(self, method_name), args, response) ret = yield from try_call_generator(getattr(self, method_name), args, response)
_ = yield from try_call_generator(self.tool_after_callback, tool_name, args, response, ret) _ = yield from try_call_generator(self.tool_after_callback, tool_name, args, response, ret)
return ret return ret
elif tool_name == 'bad_json': elif tool_name == 'bad_json': return StepOutcome(None, next_prompt=args.get('msg', 'bad_json'), should_exit=False)
return StepOutcome(None, next_prompt=args.get('msg', 'bad_json'), should_exit=False)
else: else:
yield f"未知工具: {tool_name}\n" yield f"未知工具: {tool_name}\n"
return StepOutcome(None, next_prompt=f"未知工具 {tool_name}", should_exit=False) return StepOutcome(None, next_prompt=f"未知工具 {tool_name}", should_exit=False)
@@ -43,9 +42,6 @@ def get_pretty_json(data):
data = data.copy(); data["script"] = data["script"].replace("; ", ";\n ") data = data.copy(); data["script"] = data["script"].replace("; ", ";\n ")
return json.dumps(data, indent=2, ensure_ascii=False).replace('\\n', '\n') return json.dumps(data, indent=2, ensure_ascii=False).replace('\\n', '\n')
_TOOL_ICONS = {'file_read': '📖', 'file_write': '✏️', 'file_patch': '✏️', 'code_run': '⚙️',
'web_scan': '🌐', 'web_execute_js': '🌐', 'update_working_checkpoint': '💾', 'ask_user': '', 'start_long_term_update': '💾'}
def agent_runner_loop(client, system_prompt, user_input, handler, tools_schema, max_turns=40, verbose=True, initial_user_content=None): def agent_runner_loop(client, system_prompt, user_input, handler, tools_schema, max_turns=40, verbose=True, initial_user_content=None):
messages = [ messages = [
{"role": "system", "content": system_prompt}, {"role": "system", "content": system_prompt},
@@ -72,11 +68,10 @@ def agent_runner_loop(client, system_prompt, user_input, handler, tools_schema,
tool_results = []; next_prompts = set(); should_exit = None tool_results = []; next_prompts = set(); should_exit = None
for ii, tc in enumerate(tool_calls): for ii, tc in enumerate(tool_calls):
tool_name, args, tid = tc['tool_name'], tc['args'], tc.get('id', '') tool_name, args, tid = tc['tool_name'], tc['args'], tc.get('id', '')
icon = _TOOL_ICONS.get(tool_name, '🛠️')
if tool_name == 'no_tool': pass if tool_name == 'no_tool': pass
else: else:
if verbose: yield f"{icon} 正在调用工具: `{tool_name}` 📥参数:\n````text\n{get_pretty_json(args)}\n````\n" if verbose: yield f"🛠️ 正在调用工具: `{tool_name}` 📥参数:\n````text\n{get_pretty_json(args)}\n````\n"
else: yield f"{icon} {tool_name}({_compact_tool_args(tool_name, args)})\n\n\n" else: yield f"🛠️ {tool_name}({_compact_tool_args(tool_name, args)})\n\n\n"
handler.current_turn = turn handler.current_turn = turn
gen = handler.dispatch(tool_name, args, response, index=ii) gen = handler.dispatch(tool_name, args, response, index=ii)
try: try:

View File

@@ -32,6 +32,27 @@ async function handleExtMessage(msg, sender) {
} }
} catch (e) { return { ok: false, error: e.message }; } } catch (e) { return { ok: false, error: e.message }; }
} }
if (msg.cmd === 'management') {
try {
if (msg.method === 'list') {
const all = await chrome.management.getAll();
return { ok: true, data: all.map(e => ({ id: e.id, name: e.name, enabled: e.enabled, type: e.type, version: e.version })) };
}
if (msg.method === 'reload') {
chrome.alarms.create('tmwd-self-reload', { when: Date.now() + 200 });
return { ok: true };
}
if (msg.method === 'disable') {
await chrome.management.setEnabled(msg.extId, false);
return { ok: true };
}
if (msg.method === 'enable') {
await chrome.management.setEnabled(msg.extId, true);
return { ok: true };
}
return { ok: false, error: 'Unknown method: ' + msg.method };
} catch (e) { return { ok: false, error: e.message }; }
}
return { ok: false, error: 'Unknown cmd: ' + msg.cmd }; return { ok: false, error: 'Unknown cmd: ' + msg.cmd };
} }
@@ -136,11 +157,12 @@ function buildExecScript(code, errorHandler) {
const lastLine = lines.length > 0 ? lines[lines.length - 1].trim() : ''; const lastLine = lines.length > 0 ? lines[lines.length - 1].trim() : '';
const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor; const AsyncFunction = Object.getPrototypeOf(async function(){}).constructor;
let r; let r;
function _air(c) { const ls = c.split(/\\r?\\n/); let i = ls.length - 1; while (i >= 0 && !ls[i].trim()) i--; if (i < 0) return c; const t = ls[i].trim(); if (/^(return |return;|return$|let |const |var |if |if\\(|for |for\\(|while |while\\(|switch|try |throw |class |function |async |import |export |\\/\\/|})/.test(t)) return c; ls[i] = ls[i].match(/^(\\s*)/)[1] + 'return ' + t; return ls.join('\\n'); }
if (lastLine.startsWith('return')) { if (lastLine.startsWith('return')) {
r = await (new AsyncFunction(jsCode))(); r = await (new AsyncFunction(jsCode))();
} else { } else {
try { r = eval(jsCode); if (r instanceof Promise) r = await r; } catch (e) { try { r = eval(jsCode); if (r instanceof Promise) r = await r; } catch (e) {
if (e instanceof SyntaxError && (/return/i.test(e.message) || /await/i.test(e.message))) { r = await (new AsyncFunction(jsCode))(); } else throw e; if (e instanceof SyntaxError && (/return/i.test(e.message) || /await/i.test(e.message))) { r = await (new AsyncFunction(_air(jsCode)))(); } else throw e;
} }
} }
return { ok: true, data: smartProcessResult(r) }; return { ok: true, data: smartProcessResult(r) };
@@ -190,6 +212,10 @@ async function isServerAlive() {
} }
chrome.alarms.onAlarm.addListener(async (alarm) => { chrome.alarms.onAlarm.addListener(async (alarm) => {
if (alarm.name === 'tmwd-self-reload') {
chrome.runtime.reload();
return;
}
if (alarm.name === 'tmwd-ws-keepalive') { if (alarm.name === 'tmwd-ws-keepalive') {
// Keepalive: ping to keep SW alive + detect dead connections // Keepalive: ping to keep SW alive + detect dead connections
if (ws && ws.readyState === WebSocket.OPEN) { if (ws && ws.readyState === WebSocket.OPEN) {
@@ -220,8 +246,11 @@ async function handleWsExec(data) {
ws.send(JSON.stringify({ type: 'error', id: data.id, error: 'No tabId provided' })); ws.send(JSON.stringify({ type: 'error', id: data.id, error: 'No tabId provided' }));
return; return;
} }
// Use onCreated listener to reliably capture new tabs (avoids race condition with query-diff)
const newTabIds = new Set();
const onCreated = (tab) => { newTabIds.add(tab.id); };
chrome.tabs.onCreated.addListener(onCreated);
try { try {
const tabsBefore = new Set((await chrome.tabs.query({})).map(t => t.id));
let res; let res;
try { try {
const result = await chrome.scripting.executeScript({ const result = await chrome.scripting.executeScript({
@@ -260,7 +289,14 @@ async function handleWsExec(data) {
res = { ok: false, error: { name: 'Error', message: 'CDP fallback failed: ' + cdpErr.message, stack: '' } }; res = { ok: false, error: { name: 'Error', message: 'CDP fallback failed: ' + cdpErr.message, stack: '' } };
} }
} }
const newTabs = (await chrome.tabs.query({})).filter(t => !tabsBefore.has(t.id)).map(t => ({id: t.id, url: t.url, title: t.title})); // Grace period for async tab creation (e.g. link click with target=_blank)
if (newTabIds.size === 0) await new Promise(r => setTimeout(r, 200));
chrome.tabs.onCreated.removeListener(onCreated);
// Get full info for captured new tabs
const newTabs = [];
for (const id of newTabIds) {
try { const t = await chrome.tabs.get(id); newTabs.push({id: t.id, url: t.url, title: t.title}); } catch (_) {}
}
if (res?.ok) { if (res?.ok) {
ws.send(JSON.stringify({ type: 'result', id: data.id, result: res.data, newTabs })); ws.send(JSON.stringify({ type: 'result', id: data.id, result: res.data, newTabs }));
} else { } else {
@@ -269,6 +305,8 @@ async function handleWsExec(data) {
} }
} catch (e) { } catch (e) {
ws.send(JSON.stringify({ type: 'error', id: data.id, error: { name: e.name || 'Error', message: e.message || String(e), stack: e.stack || '' } })); ws.send(JSON.stringify({ type: 'error', id: data.id, error: { name: e.name || 'Error', message: e.message || String(e), stack: e.stack || '' } }));
} finally {
chrome.tabs.onCreated.removeListener(onCreated);
} }
} }

View File

@@ -10,7 +10,8 @@
"debugger", "debugger",
"scripting", "scripting",
"alarms", "alarms",
"declarativeNetRequest" "declarativeNetRequest",
"management"
], ],
"host_permissions": ["<all_urls>"], "host_permissions": ["<all_urls>"],
"background": { "background": {

View File

@@ -35,7 +35,7 @@
}}, }},
{"type": "function", "function": { {"type": "function", "function": {
"name": "web_scan", "name": "web_scan",
"description": "Get simplified HTML and tab list. Filters sidebars/floating elements; use execute_js for filtered content. Call after switching pages", "description": "Get simplified HTML and tab list. Removes hidden/floating/covered elements. Call after switching pages",
"parameters": {"type": "object", "properties": { "parameters": {"type": "object", "properties": {
"tabs_only": {"type": "boolean", "description": "Show tab list only, no HTML", "default": false}, "tabs_only": {"type": "boolean", "description": "Show tab list only, no HTML", "default": false},
"switch_tab_id": {"type": "string", "description": "[Optional] Tab ID to switch to"}, "switch_tab_id": {"type": "string", "description": "[Optional] Tab ID to switch to"},
@@ -43,7 +43,7 @@
}}, }},
{"type": "function", "function": { {"type": "function", "function": {
"name": "web_execute_js", "name": "web_execute_js",
"description": "Execute JS to control browser. Use precisely to reduce web_scan calls. Put code in ```javascript blocks in reply body to avoid escaping", "description": "Execute JS to control browser. No guessing. Act accurately to reduce web_scan calls. Put code in ```javascript blocks in reply body to avoid escaping",
"parameters": {"type": "object", "properties": { "parameters": {"type": "object", "properties": {
"script": {"type": "string", "description": "[Mutually exclusive] JS code or script path. NEVER use this param when use reply code block"}, "script": {"type": "string", "description": "[Mutually exclusive] JS code or script path. NEVER use this param when use reply code block"},
"save_to_file": {"type": "string", "description": "file path; **only** for long result", "default": ""}, "save_to_file": {"type": "string", "description": "file path; **only** for long result", "default": ""},

View File

@@ -35,7 +35,7 @@
}}, }},
{"type": "function", "function": { {"type": "function", "function": {
"name": "web_scan", "name": "web_scan",
"description": "获取当前页面的简化HTML内容和标签页列表。注意简化会过滤边栏、浮动元素等非主体内容如需查看被过滤内容请用execute_js。切换页面后一般应先调用查看", "description": "获取当前页面的简化HTML内容和标签页列表。会移除隐藏/浮动/被遮盖的元素。切换页面后一般应先调用查看",
"parameters": {"type": "object", "properties": { "parameters": {"type": "object", "properties": {
"tabs_only": {"type": "boolean", "description": "仅返回标签页列表和当前标签信息不获取HTML内容", "default": false}, "tabs_only": {"type": "boolean", "description": "仅返回标签页列表和当前标签信息不获取HTML内容", "default": false},
"switch_tab_id": {"type": "string", "description": "可选的标签页 ID。如果提供系统将在扫描前切换到该标签页"}, "switch_tab_id": {"type": "string", "description": "可选的标签页 ID。如果提供系统将在扫描前切换到该标签页"},
@@ -43,7 +43,7 @@
}}, }},
{"type": "function", "function": { {"type": "function", "function": {
"name": "web_execute_js", "name": "web_execute_js",
"description": "执行 JS 控制浏览器。建议精准使用减少 web_scan。为免转义问题代码优先考虑放回复正文 ```javascript 块", "description": "执行 JS 控制浏览器。禁止猜测,准确操作以减少 web_scan 调用。为免转义问题,代码优先考虑放回复正文 ```javascript 块",
"parameters": {"type": "object", "properties": { "parameters": {"type": "object", "properties": {
"script": {"type": "string", "description": "[Optional] JS代码或路径。为免转义建议留空改用正文代码块与此参数互斥"}, "script": {"type": "string", "description": "[Optional] JS代码或路径。为免转义建议留空改用正文代码块与此参数互斥"},
"save_to_file": {"type": "string", "description": "结果存文件,适合返回值较长时", "default": ""}, "save_to_file": {"type": "string", "description": "结果存文件,适合返回值较长时", "default": ""},

View File

@@ -2,26 +2,18 @@
- 禁止import直接用web_scan/web_execute_js工具。本文件只记录特性和坑。 - 禁止import直接用web_scan/web_execute_js工具。本文件只记录特性和坑。
- 底层:`../TMWebDriver.py`通过Chrome扩展(非Tampermonkey)接管用户浏览器(保留登录态/Cookie - 底层:`../TMWebDriver.py`通过Chrome扩展(非Tampermonkey)接管用户浏览器(保留登录态/Cookie
- 非Selenium/Playwright不需调试浏览器或新数据目录 - 非Selenium/Playwright保留用户浏览器登录态
- 支撑 `web_scan`(只读DOM) / `web_execute_js`(执行JS) 等高层工具 - ⚠扩展更新后旧tab的content script不重载→需刷新页面
- ⚠扩展更新后已打开的旧tab不会自动加载新版脚本→scan/execute_js无ACK→需刷新页面或切到新tab
## 通用特性 ## 通用特性
- web_execute_js**完美支持顶层await**v0.4+),可直接`await fetch()`/`await new Promise()` - web_execute_js里使用`await`时需**显式`return`**才能拿到返回值底层async包裹不写return则返回null
- ⚠使用await时需**显式`return`**才能拿到返回值底层async包裹不写return则返回null - ✅web_scan自动穿透同源iframe跨域iframe需CDP或postMessage见下方章节
- ✅web_scan**自动穿透同源iframe**无需手动操作scan直接递归输出iframe内部DOM。跨域iframe则需CDP或postMessage见下方章节
## 限制(isTrusted) ## 限制(isTrusted)
- JS dispatch的事件`isTrusted=false`,敏感操作(文件上传/部分按钮)会被浏览器拦截 - JS事件`isTrusted=false`,敏感操作(如文件上传/部分按钮)可能被拦截;这类场景首选**CDP桥**
- ⭐**首选绕过CDP桥**——CDP派发的Input事件是浏览器原生级别(isTrusted=true)且无需前台见下方CDP章节 - ⚠JS点击按钮打不开新tab→可能是浏览器弹窗拦截换CDP点击试试
- 文件上传JS无法填充`<input type=file>` - 文件上传JS无法填充`<input type=file>`首选CDP batchgetDocument→querySelector→DOM.setFileInputFiles备选ljqCtrl物理点击
- ⭐首选CDP batchgetDocument→querySelector→DOM.setFileInputFiles(无需前台/物理点击) - 需转物理坐标时:`physX = (screenX + rect中心x) * dpr``physY = (screenY + chromeH + rect中心y) * dpr`;其中 `chromeH = outerHeight - innerHeight`
- 备选ljqCtrl物理点击SetForegroundWindow→点上传按钮→FindWindow轮询对话框→输入路径→轮询关闭
- 备选:元素→屏幕物理坐标(ljqCtrl/PostMessage点击前必算)JS一次取rect+窗口信息,公式:
- `physX = (screenX + rect中心x) * dpr``physY = (screenY + chromeH + rect中心y) * dpr`
- chromeH = outerHeight - innerHeightdpr = devicePixelRatio
- 注意screenX/Y也是CSS像素所有值先加后统一乘dpr
- 结论:读信息+普通操作用TMWebDriver需isTrusted事件首选CDP桥文件上传首选CDP三连(备选ljqCtrl)
## 导航 ## 导航
- `web_scan` 仅读当前页不导航,切换网站用 `web_execute_js` + `location.href='url'` - `web_scan` 仅读当前页不导航,切换网站用 `web_execute_js` + `location.href='url'`
@@ -45,14 +37,11 @@ fetch('PDF_URL').then(r=>r.blob()).then(b=>{
注意需同源或CORS允许跨域先导航到目标域再执行 注意需同源或CORS允许跨域先导航到目标域再执行
## Chrome后台标签节流 ## Chrome后台标签节流
- 后台标签中`setTimeout`被Chrome intensive throttling延迟到≥1min/次 - 后台标签中`setTimeout`被Chrome intensive throttling延迟到≥1min/次扩展脚本中避免依赖setTimeout轮询
- 扩展content script中detect_newtab的轮询(`setTimeout 150ms × 10`)会超时
- 已修复移除脚本内轮询改由Python侧`get_session_dict()`前后对比检测新标签
- 同理扩展脚本中任何后台逻辑都应避免依赖setTimeout轮询
## CDP桥(tmwd_cdp_bridge扩展) ⭐首选 ## CDP桥(tmwd_cdp_bridge扩展) ⭐首选
扩展路径:`assets/tmwd_cdp_bridge/`(需安装含debugger权限) 扩展路径:`assets/tmwd_cdp_bridge/`(需安装含debugger权限)
⚠TID约定标识(非密钥):首次运行自动生成到`assets/tmwd_cdp_bridge/config.js`(已gitignore)扩展通过manifest引用 ⚠TID约定标识首次运行自动生成到`assets/tmwd_cdp_bridge/config.js`(已gitignore)扩展通过manifest引用
调用:`web_execute_js` script直传JSON字符串工具层自动识别对象格式走WS→background.js cmd路由 调用:`web_execute_js` script直传JSON字符串工具层自动识别对象格式走WS→background.js cmd路由
```js ```js
// 直接传JSON字符串作为script参数无需DOM操作 // 直接传JSON字符串作为script参数无需DOM操作
@@ -62,43 +51,23 @@ web_execute_js script='{"cmd": "cdp", "tabId": N, "method": "...", "params": {..
web_execute_js script='{"cmd": "batch", "commands": [...]}' web_execute_js script='{"cmd": "batch", "commands": [...]}'
// 返回值直接是JSON结果 // 返回值直接是JSON结果
``` ```
⚠旧DOM方式(TID元素+MutationObserver)仍可用但已不推荐 通信方式⭐JSON字符串直传(首选) | TID DOM方式(TID元素+MutationObserverweb_scan/execute_js底层依赖)
单命令:`{cmd:'tabs'}` | `{cmd:'cookies'}` | `{cmd:'cdp', tabId:N, method:'...', params:{...}}` 单命令:`{cmd:'tabs'}` | `{cmd:'cookies'}` | `{cmd:'cdp', tabId:N, method:'...', params:{...}}` | `{cmd:'management', method:'list|reload|disable|enable', extId:'...'}`
- managementlist返回所有扩展信息reload/disable/enable需传extId
- ⭐batch混合`{cmd:'batch', commands:[{cmd:'cookies'},{cmd:'tabs'},{cmd:'cdp',...},...]}` - ⭐batch混合`{cmd:'batch', commands:[{cmd:'cookies'},{cmd:'tabs'},{cmd:'cdp',...},...]}`
- 返回`{ok:true, results:[...]}`一次请求多命令CDP懒attach复用session - 返回`{ok:true, results:[...]}`一次请求多命令CDP懒attach复用session
- 子命令会自动继承外层batch的tabId如cookies命令可正确获取当前页面URL - 子命令会自动继承外层batch的tabId如cookies命令可正确获取当前页面URL
- `$N.path`引用第N个结果字段(0-indexed),如`"nodeId":"$2.root.nodeId"` - `$N.path`引用第N个结果字段(0-indexed),如`"nodeId":"$2.root.nodeId"`
- ⚠batch前序命令失败时后续`$N`引用拿到undefined,整条链路**静默失败不报错**,需检查返回results数组中每项的ok状态未验证BBS#46 - ⚠batch前序命令失败时后续`$N`引用会静默变成undefined;要检查results数组中每项的ok状态
- 典型文件上传三连 getDocument(**depth:1**性能优化200ms+→个位数ms)→querySelector(input[type=file])→setFileInputFiles未验证BBS#38 - 典型文件上传getDocument(**depth:1**) → querySelector(`input[type=file]`) → setFileInputFiles
- ⚠nodeId路径一致性getDocument+querySelector路径和performSearch+getSearchResults路径的nodeId**不互通**同一batch内不可混用未验证BBS#45 - 思想:
- ⚠文件上传后前端框架(React/Vue)可能不感知→JS补发**两个事件**Vue3需input事件而非仅change未验证BBS#35/#39 - 同一链路内保持nodeId来源一致不混用querySelector路径与performSearch路径
```js - 上传后前端框架可能不感知必要时JS补发`input`/`change`事件
el.dispatchEvent(new Event('input', {bubbles:true})); - 上传前检查`input.accept`多input时用accept/父容器语义区分
el.dispatchEvent(new Event('change', {bubbles:true})); - 等待元素优先用`DOM.performSearch('input[type=file]')`做轻量轮询
``` - 瞬态input的核心是**缩短发现→setFileInputFiles时间窗**优先同batch完成再不行用DOM事件监听猴子补丁仅作兜底思路
- Electron<12/旧WebView可能无InputEvent构造函数防御性降级未验证BBS#42
`const Ctor = typeof InputEvent !== 'undefined' ? InputEvent : Event; el.dispatchEvent(new Ctor('input', {bubbles:true}));`
- 极端情况(框架仍不响应)Runtime.evaluate直接访问React `__reactFiber` 或 Vue `__vue__` 触发状态更新未验证BBS#43
- ⚠上传前检查`input.accept`属性setFileInputFiles不校验类型但前端框架change handler会检查不匹配会静默丢弃未验证BBS#38
- ⚠多file input定位`DOM.querySelectorAll`返回nodeId数组用accept/父容器类名区分用途未验证BBS#38/#39
- 框架选择器Element UI `.el-upload__input` | Ant Design `.ant-upload input[type=file]` | Naive UI `.n-upload-trigger input[type=file]` | Dropzone `.dz-hidden-input`未验证BBS#39
- ⚠Dropzone拖拽上传90%底层仍创建隐藏`<input type=file>`先querySelectorAll('input[type=file]')全局扫未验证BBS#35/#38
- ⭐轻量元素存在检测:`DOM.performSearch({query:'input[type=file]'})`返回resultCount不触发DOM树构建轮询等待元素时避免重复getDocument未验证BBS#39
- performSearch支持三种语法CSS选择器 / XPath(`//input[@type='file']`) / 纯文本自动识别未验证BBS#41
- ⭐瞬态file input处理Ant Design等框架点击上传按钮时动态创建input上传完立即销毁未验证BBS#42/#43
- 方案A(批处理)在同一batch内完成 performSearch→getSearchResults→setFileInputFiles→**discardSearchResults**缩小input被销毁的时间窗口discardSearchResults防searchId泄漏未验证BBS#46
- 方案B(事件监听)`DOM.enable`后监听`DOM.childNodeInserted`事件捕获input创建瞬间零延迟拿到nodeId
- 前提须先对document.body的nodeId调`DOM.requestChildNodes`否则CDP不推送子树变更
- ⚠`DOM.disable`会使所有已获取nodeId失效setFileInputFiles必须在disable之前。正确时序DOM.enable→requestChildNodes→[等事件]→setFileInputFiles→DOM.disable未验证BBS#45
- 方案C(猴子补丁兜底)Runtime.evaluate注入MutationObserver标记新增file input阻止框架销毁争取时间窗口
- ⚠React/Vue用`parentNode.removeChild(node)`而非`node.remove()`需patch `Element.prototype.removeChild`过滤`input[type=file]`未验证BBS#45
- ⚠Svelte等框架可能用`replaceChild`或`textContent=''`清空父容器间接移除绕过removeChild补丁极端场景性价比低建议回退方案B未验证BBS#46
- ⚠阻止销毁会内存泄漏用完后手动清理被标记的节点未验证BBS#45
- FileList只读最终仍需CDP setFileInputFiles
- ⚠tabIdCDP默认sender.tab.id(当前注入页)跨tab需显式tabId或先batch内tabs查 - ⚠tabIdCDP默认sender.tab.id(当前注入页)跨tab需显式tabId或先batch内tabs查
- CDP可用任意方法(Input/Network/DOM/Page/Runtime/Emulation等)单条每次attach→send→detach
- ⭐跨tab无需前台指定tabId即可操作后台标签页 - ⭐跨tab无需前台指定tabId即可操作后台标签页
- ⭐绕过isTrustedCDP派发的Input事件是浏览器原生级别
## CDP点击完整生命周期未验证BBS#23 ## CDP点击完整生命周期未验证BBS#23
- 通用点击需**三事件序列**mouseMoved → mousePressed → mouseReleased间隔50-100ms - 通用点击需**三事件序列**mouseMoved → mousePressed → mouseReleased间隔50-100ms
@@ -118,10 +87,8 @@ web_execute_js script='{"cmd": "batch", "commands": [...]}'
- postMessage中继方案仅在content script已注入iframe时有效第三方支付iframe通常无注入 - postMessage中继方案仅在content script已注入iframe时有效第三方支付iframe通常无注入
## CDP文本输入未验证BBS#23 ## CDP文本输入未验证BBS#23
- `Input.insertText({text:'...'})` — 直接插入不触发keydown/keyup - `insertText`快但无key事件受控组件需补dispatch `input`事件
- `Input.dispatchKeyEvent`逐键派发,慢但完整模拟 - 需完整键盘模拟时用`dispatchKeyEvent`逐键派发
- React/Vue受控组件先insertText再JS手动dispatch `input`事件input事件不检查isTrusted
- 简单输入框用insertText够用
## CDP DOM域穿透 closed Shadow DOM未验证BBS#24/#25 ## CDP DOM域穿透 closed Shadow DOM未验证BBS#24/#25
- `DOM.getDocument({depth:-1, pierce:true})` 穿透所有Shadow边界含closed - `DOM.getDocument({depth:-1, pierce:true})` 穿透所有Shadow边界含closed
@@ -130,49 +97,29 @@ web_execute_js script='{"cmd": "batch", "commands": [...]}'
- ⚠不能简化为对角线平均——元素有transform:rotate/skew时四点非矩形 - ⚠不能简化为对角线平均——元素有transform:rotate/skew时四点非矩形
- querySelector**不能跨Shadow边界写组合选择器**需分步先找host再在其shadow内找子元素 - querySelector**不能跨Shadow边界写组合选择器**需分步先找host再在其shadow内找子元素
- ⚠nodeId在DOM变更后失效 → 用`backendNodeId`更稳定或重新getDocument刷新 - ⚠nodeId在DOM变更后失效 → 用`backendNodeId`更稳定或重新getDocument刷新
- 渲染检查:`DOM.resolveNode` → `Runtime.callFunctionOn` 检查offsetHeight>0
- 完整pipeline: getDocument(pierce) → querySelector → getBoxModel → 四点平均坐标 → Input三事件点击
## autofill获取与登录 (需 v0.4+ 脚本支持 await)
## autofill获取与登录
检测web_scan输出input带`data-autofilled="true"`value显示为受保护提示(非真实值Chrome安全保护需点击释放) 检测web_scan输出input带`data-autofilled="true"`value显示为受保护提示(非真实值Chrome安全保护需点击释放)
- ⚠**前置条件必须先CDP `Page.bringToFront` 切tab到前台**Chrome仅在前台tab释放autofill保护值后台tab物理点击无效 - ⚠**前置条件必须先CDP `Page.bringToFront` 切tab到前台**Chrome仅在前台tab释放autofill保护值后台tab物理点击无效
- ⭐**一键释放与登录**利用顶层 `await`,在单次 `web_execute_js` 中连贯完成: - ⭐**一键释放与登录**bringToFront → mousePressed点任一字段(无需Released一个释放全页) → 等500ms → 补input/change事件 → 点登录
1. CDP batch发送 `Page.bringToFront` 切到前台。
2. JS获取输入框坐标。
3. CDP发送 `Input.dispatchMouseEvent` (mousePressed) 物理点击释放autofill。
4. `await new Promise(r => setTimeout(r, 500))` 等待释放。
5. 派发 `input`/`change` 事件唤醒前端框架(解禁登录按钮)。
6. 触发登录点击。
- ⚠只需 `mousePressed`,无需 `mouseReleased`。点击一个字段即释放全页。
- ⚠使用await时需显式`return`返回值否则async包裹层默认返回null。
## 验证码/页面视觉截图 ## 验证码/页面视觉截图
- ⭐首选CDP截图`Page.captureScreenshot`(format:'png')→返回base64无需前台/后台tab也行全页高清 - ⭐首选CDP截图`Page.captureScreenshot`(format:'png')→返回base64无需前台/后台tab也行全页高清
- 验证码canvas/imgJS `canvas.toDataURL()` 直接拿base64最干净 - 验证码canvas/imgJS `canvas.toDataURL()` 直接拿base64最干净
- 备选:`window.open(location.href,'_blank')` 前台开新标签→win32截图→完后close
## simphtml与TMWebDriver调试 ## simphtml与TMWebDriver调试
- ⭐**simphtml唯一调试方法**必须通过 `code_run` 注入JS到真实浏览器执行,因为Python端无法完全模拟DOM行为。 - simphtml调试必须通过`code_run`注入JS到真实浏览器Python端无法模拟DOM
```python - `d=TMWebDriver()`, `d.set_session('url_pattern')`, `d.execute_js(code)` → 返回`{'data': value}`
import sys - simphtml`str(simphtml.optimize_html_for_tokens(html))` — 返回BS4 Tag需str()
sys.path.append('../') - ⚠**DOMRect坑(hasOverlap)**:某些上下文`rect.x/y`为undefined(只有left/top)导致NaN→误判重叠。兼容`rect.x ?? rect.left`
from TMWebDriver import *
from simphtml import *
driver = TMWebDriver()
res = driver.execute_js(js_optHTML) # js_optHTML为simphtml中注入的JS代码
```
- `d=TMWebDriver()`, `d.set_session('url_pattern')`, `d.execute_js('code')` → 返回`{'data': value}`(非裸值)
- 配合simphtml`str(simphtml.optimize_html_for_tokens(html))` → 注意返回BS4 Tag需str()
- ⚠**DOMRect坑点(hasOverlap)**`DOMRect` 对象在某些浏览器/上下文中可能缺少 `x` 和 `y` 属性(只有 `left`/`top`),直接访问 `rect.x` 会得到 `undefined`,导致数学计算(如重叠判定)变成 `NaN`,从而引发逻辑错误(如错误判定为重叠导致元素被误删)。必须兼容:`const x = rect.x !== undefined ? rect.x : rect.left;`
## 跨域iframe操控(postMessage中继) ## 跨域iframe操控(postMessage中继)
- 跨域iframe的contentDocument不可访问web_execute_js只在顶层执行 - 扩展content script支持顶层发postMessage到iframeiframe内eval+回传结果
- 扩展content script已支持iframe内不return改为监听postMessage并eval执行+回传结果
- 顶层发送:`iframe.contentWindow.postMessage({type:'ljq_exec', id, code}, '*')` - 顶层发送:`iframe.contentWindow.postMessage({type:'ljq_exec', id, code}, '*')`
- iframe回传`{type:'ljq_result', id, result}` 通过window.addEventListener('message')接收 - iframe回传`{type:'ljq_result', id, result}` 通过window.addEventListener('message')接收
- ⚠只能eval表达式不支持return/函数体包装,构造代码时注意 - ⚠只能eval表达式不支持return/函数体包装
- 流程发postMessage→等→读window._ljqResults[id]获取结果 - 已验证读取iframe内DOM、填写input均成功
- 已验证读取iframe内DOM(document.title)、填写input均成功
## 连不上排查 ## 连不上排查
web_scan失败时按序排查 web_scan失败时按序排查
@@ -180,6 +127,3 @@ web_scan失败时按序排查
没找到→走web_setup_sop找到→确认已启用 没找到→走web_setup_sop找到→确认已启用
②浏览器没开?→检查①对应的浏览器进程是否在跑(tasklist/ps)没有则启动并打开正常URL⚠about:blank等内部页不加载扩展 ②浏览器没开?→检查①对应的浏览器进程是否在跑(tasklist/ps)没有则启动并打开正常URL⚠about:blank等内部页不加载扩展
③WS后台挂了→socket.connect_ex(('127.0.0.1',18766))非0即dead→手动`from TMWebDriver import TMWebDriver; TMWebDriver()`起master ③WS后台挂了→socket.connect_ex(('127.0.0.1',18766))非0即dead→手动`from TMWebDriver import TMWebDriver; TMWebDriver()`起master
## 性能
- ⚠ URL必须用`127.0.0.1`不用`localhost`。Windows下localhost先尝试IPv6(::1)超时2s再回退IPv4每次HTTP请求多2s

View File

@@ -824,7 +824,7 @@ def execute_js_rich(script, driver, no_monitor=False):
try: try:
print(f"Executing: {script[:250]} ...") print(f"Executing: {script[:250]} ...")
response = driver.execute_js(script) response = driver.execute_js(script)
result = response.get('data') or response.get('result') result = response['data'] if 'data' in response else response.get('result')
if response.get('closed', 0) == 1: reloaded = True if response.get('closed', 0) == 1: reloaded = True
time.sleep(1) time.sleep(1)
except Exception as e: except Exception as e:
@@ -835,10 +835,10 @@ def execute_js_rich(script, driver, no_monitor=False):
rr = { rr = {
"status": "failed" if error_msg else "success", "status": "failed" if error_msg else "success",
"js_return": result, "js_return": result,
"environment": {"reloaded": reloaded},
"tab_id": driver.default_session_id "tab_id": driver.default_session_id
} }
if response.get('newTabs'): rr['environment']['newTabs'] = response['newTabs'] if reloaded: rr['reloaded'] = reloaded
if response.get('newTabs'): rr['newTabs'] = response['newTabs']
else: else:
after = driver.get_session_dict() after = driver.get_session_dict()
new_sids = {k: v for k, v in after.items() if k not in before_sids} new_sids = {k: v for k, v in after.items() if k not in before_sids}