Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/AI/image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,19 @@ export class Image {
return `[CQ:image,file=${file}]`;
}

get base64Url(): string {
let format = this.format;
if (!format || format === "unknown") format = 'png';
return `data:image/${format};base64,${this.base64}`
}

/**
* 获取图片的URL,若为base64则返回base64Url
*/
get url(): string {
return this.type === 'base64' ? this.base64Url : this.file;
}

async checkImageUrl(): Promise<boolean> {
if (this.type !== 'url') return true;
let isValid = false;
Expand Down Expand Up @@ -111,7 +124,7 @@ export class Image {
role: "user",
content: [{
"type": "image_url",
"image_url": { "url": this.type === 'base64' ? `data:image/${this.format};base64,${this.base64}` : this.file }
"image_url": { "url": this.url }
}, {
"type": "text",
"text": prompt ? prompt : defaultPrompt
Expand All @@ -123,7 +136,7 @@ export class Image {
if (!this.content && urlToBase64 === '自动' && this.type === 'url') {
logger.info(`图片${this.id}第一次识别失败,自动尝试使用转换为base64`);
await this.urlToBase64();
messages[0].content[0].image_url.url = `data:image/${this.format};base64,${this.base64}`;
messages[0].content[0].image_url.url = this.base64Url;
this.content = (await sendITTRequest(messages)).slice(0, maxChars);
}

Expand Down
7 changes: 5 additions & 2 deletions src/tool/tool_image.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ export function registerImage() {
properties: {
id: {
type: "string",
description: `图片的id,六位字符,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
description: `图片id,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
},
content: {
type: "string",
Expand Down Expand Up @@ -74,4 +74,7 @@ export function registerImage() {
return { content: `图像生成失败:${e}`, images: [] };
}
}
}
}

// TODO: tti改为返回图片base64
// 注意兼容问题
2 changes: 1 addition & 1 deletion src/tool/tool_meme.ts
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ export function registerMeme() {
image_ids: {
type: "array",
items: { type: "string" },
description: `图片的id,六位字符,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
description: `图片id,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
},
save: {
type: "boolean",
Expand Down
69 changes: 56 additions & 13 deletions src/tool/tool_render.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
import { logger } from "../logger";
import { Tool } from "./tool";
import { ConfigManager } from "../config/configManager";
import { AIManager } from "../AI/AI";
import { AI, AIManager } from "../AI/AI";
import { Image } from "../AI/image";
import { generateId } from "../utils/utils";
import { parseSpecialTokens } from "../utils/utils_string";

interface RenderResponse {
status: string;
Expand Down Expand Up @@ -33,14 +34,53 @@ async function postToRenderEndpoint(endpoint: string, bodyData: any): Promise<Re
}
}

async function transformContentToUrlText(ctx: seal.MsgContext, ai: AI, content: string): Promise<{ text: string, images: Image[] }> {
const segs = parseSpecialTokens(content);
let text = '';
const images: Image[] = [];
for (const seg of segs) {
switch (seg.type) {
case 'text': {
text += seg.content;
break;
}
case 'at': {
const name = seg.content;
const ui = await ai.context.findUserInfo(ctx, name);
if (ui !== null) {
text += ` @${ui.name} `;
} else {
logger.warning(`无法找到用户:${name}`);
text += ` @${name} `;
}
break;
}
case 'img': {
const id = seg.content;
const image = await ai.context.findImage(ctx, id);

if (image) {
if (image.type === 'local') throw new Error(`图片<|img:${id}|>为本地图片,暂不支持`);
images.push(image);
text += image.url;
} else {
logger.warning(`无法找到图片:${id}`);
}
break;
}
}
}
return { text, images };
}

// Markdown 渲染
async function renderMarkdown(markdown: string, theme: 'light' | 'dark' | 'gradient' = 'light', width = 1200) {
return postToRenderEndpoint('/render/markdown', { markdown, theme, width, quality: 90 });
async function renderMarkdown(markdown: string, theme: 'light' | 'dark' | 'gradient' = 'light', width = 1200, hasImages = false) {
return postToRenderEndpoint('/render/markdown', { markdown, theme, width, quality: 90, hasImages });
}

// HTML 渲染
async function renderHtml(html: string, width = 1200) {
return postToRenderEndpoint('/render/html', { html, width, quality: 90 });
async function renderHtml(html: string, width = 1200, hasImages = false) {
return postToRenderEndpoint('/render/html', { html, width, quality: 90, hasImages });
}

export function registerRender() {
Expand All @@ -54,7 +94,7 @@ export function registerRender() {
properties: {
content: {
type: "string",
description: "要渲染的 Markdown 内容。支持 LaTeX 数学公式,使用前后 $ 包裹行内公式,前后 $$ 包裹块级公式"
description: "要渲染的 Markdown 内容。支持 LaTeX 数学公式,使用前后 $ 包裹行内公式,前后 $$ 包裹块级公式。可以使用<|img:xxxxxx|>替代图片url(注意使用markdown语法显示图片),xxxxxx为" + `图片id,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
},
name: {
type: "string",
Expand Down Expand Up @@ -87,7 +127,10 @@ export function registerRender() {
const kws = ["render", "markdown", name, theme];

try {
const result = await renderMarkdown(content, theme, 1200);
const { text, images } = await transformContentToUrlText(ctx, ai, content);
const hasImages = images.length > 0;

const result = await renderMarkdown(text, theme, 1200, hasImages);
if (result.status === "success" && result.base64) {
const base64 = result.base64;
if (!base64) {
Expand Down Expand Up @@ -124,7 +167,7 @@ export function registerRender() {
properties: {
content: {
type: "string",
description: "要渲染的 HTML 内容。支持 LaTeX 数学公式,使用前后 $ 包裹行内公式,前后 $$ 包裹块级公式。"
description: "要渲染的 HTML 内容。支持 LaTeX 数学公式,使用前后 $ 包裹行内公式,前后 $$ 包裹块级公式。可以使用<|img:xxxxxx|>替代图片url(注意使用html元素显示图片),xxxxxx为" + `图片id,或user_avatar:用户名称` + (ConfigManager.message.showNumber ? '或纯数字QQ号' : '') + `,或group_avatar:群聊名称` + (ConfigManager.message.showNumber ? '或纯数字群号' : '')
},
name: {
type: "string",
Expand All @@ -151,7 +194,10 @@ export function registerRender() {
const kws = ["render", "html", name];

try {
const result = await renderHtml(content, 1200);
const { text, images } = await transformContentToUrlText(ctx, ai, content);
const hasImages = images.length > 0;

const result = await renderHtml(text, 1200, hasImages);
if (result.status === "success" && result.base64) {
const base64 = result.base64;
if (!base64) {
Expand All @@ -178,7 +224,4 @@ export function registerRender() {
}
}

// TODO:嵌入图片……
// 1. 嵌入本地图片
// 2. 嵌入网络图片,包括聊天记录,用户头像,群头像,直接使用url
// 3. 嵌入base64图片
// TODO:嵌入本地图片
4 changes: 4 additions & 0 deletions src/update.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
// 版本更新日志,格式为 "版本号": "更新内容",版本号格式为 "x.y.z",按照时间顺序从新到旧排列。
export const updateInfo = {
"4.12.1": `- 新增按时间搜索记忆
- 新增图片头像ID发送
- 将img命令改为ai子命令
- 新增render嵌入图片`,
"4.12.0": `- 新增通过名称选择角色设定功能
- 修复获取好友、群聊等列表时的bug
- 修复了调用函数时,无需cmdArgs的函数也会报错的问题
Expand Down
2 changes: 1 addition & 1 deletion src/utils/utils_string.ts
Original file line number Diff line number Diff line change
Expand Up @@ -475,7 +475,7 @@ interface TokenSegment {
content: string;
}

function parseSpecialTokens(s: string): TokenSegment[] {
export function parseSpecialTokens(s: string): TokenSegment[] {
const result: TokenSegment[] = [];
const segs = s.split(/([<<][\|│|][^::]+[::]?\s?.+?(?:[\|│|][>>]|[\|│|>>]))/);
segs.forEach(seg => {
Expand Down
18 changes: 12 additions & 6 deletions 相关后端项目/md和html图片渲染/render.js
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,8 @@ async function renderToImage(content, options = {}) {
theme = 'light',
style = 'github',
width = 1200,
quality = 90
quality = 90,
hasImages = false
} = options;

const browser = await puppeteer.launch({
Expand All @@ -287,9 +288,12 @@ async function renderToImage(content, options = {}) {

const html = generateHTML(content, contentType, theme, style);

// 如果有图片,增加超时时间(图片加载需要更长时间)
const timeout = hasImages ? 60000 : 30000;

await page.setContent(html, {
waitUntil: 'networkidle0',
timeout: 30000
timeout: timeout
});

await new Promise(r => setTimeout(r, 1500));
Expand Down Expand Up @@ -362,7 +366,7 @@ async function renderToImage(content, options = {}) {
// 渲染 Markdown
app.post('/render/markdown', async (req, res) => {
try {
const { markdown, theme = 'light', width = 1200, quality = 90 } = req.body;
const { markdown, theme = 'light', width = 1200, quality = 90, hasImages = false } = req.body;
if (!markdown) {
return res.status(400).json({ status: 'error', message: 'Field "markdown" is required' });
}
Expand All @@ -371,7 +375,8 @@ app.post('/render/markdown', async (req, res) => {
contentType: 'markdown',
theme,
width,
quality
quality,
hasImages
});

res.json({
Expand All @@ -390,15 +395,16 @@ app.post('/render/markdown', async (req, res) => {
// 渲染 HTML
app.post('/render/html', async (req, res) => {
try {
const { html, width = 1200, quality = 90 } = req.body;
const { html, width = 1200, quality = 90, hasImages = false } = req.body;
if (!html) {
return res.status(400).json({ status: 'error', message: 'Field "html" is required' });
}

const result = await renderToImage(html, {
contentType: 'html',
width,
quality
quality,
hasImages
});

res.json({
Expand Down