Skip to content

add vision #27

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
BAKAOLC merged 2 commits into from
Sep 21, 2025
Merged

add vision #27

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b11ecd5
add vision
BAKAOLC Sep 21, 2025
d503730
Update src/RitsukageBotForDiscord/Library/OpenApi/Pixiv/OpenApi.Pixiv.cs
BAKAOLC Sep 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions src/RitsukageBotForDiscord/Library/OpenApi/OpenApi.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using ZiggyCreatures.Caching.Fusion;

Expand All @@ -14,37 +16,38 @@ private OpenApi()
{
}

/// <summary>
/// Service provider
/// </summary>
public IServiceProvider? ServiceProvider { get; private set; }

/// <summary>
/// Cache provider
/// </summary>
public IFusionCache? CacheProvider { get; private set; }
public IFusionCache? CacheProvider => ServiceProvider?.GetService<IFusionCache>();

/// <summary>
/// Logger
/// Configuration
/// </summary>
public ILogger<OpenApi>? Logger { get; private set; }
public IConfiguration? Configuration => ServiceProvider?.GetService<IConfiguration>();

/// <summary>
/// Instance of OpenApi
/// Logger
/// </summary>
public static OpenApi Instance => OpenApiInstance.Value;
public ILogger<OpenApi>? Logger => ServiceProvider?.GetService<ILogger<OpenApi>>();

/// <summary>
/// Set cache provider
/// Instance of OpenApi
/// </summary>
/// <param name="cacheProvider"></param>
public void SetCacheProvider(IFusionCache cacheProvider)
{
CacheProvider = cacheProvider;
}
public static OpenApi Instance => OpenApiInstance.Value;

/// <summary>
/// Set logger
/// Set service provider
/// </summary>
/// <param name="logger"></param>
public void SetLogger(ILogger<OpenApi> logger)
/// <param name="serviceProvider"></param>
public void SetServiceProvider(IServiceProvider serviceProvider)
{
Logger = logger;
ServiceProvider = serviceProvider;
}
}
}
30 changes: 24 additions & 6 deletions src/RitsukageBotForDiscord/Library/OpenApi/Pixiv/OpenApi.Pixiv.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
using RitsukageBot.Library.OpenApi.Pixiv.Structs;
Expand Down Expand Up @@ -34,8 +35,11 @@ public static class OpenApiPixiv

httpClient ??= NetworkUtility.GetHttpClient();

if (!httpClient.DefaultRequestHeaders.Contains("Referer"))
httpClient.DefaultRequestHeaders.Add("Referer", "https://www.pixiv.net/");
httpClient.DefaultRequestHeaders.Referrer = new("https://www.pixiv.net/");

var cookieString = instance.Configuration?.GetValue<string>("OpenApi:PixivCookie");
if (!string.IsNullOrWhiteSpace(cookieString))
httpClient.DefaultRequestHeaders.Add("Cookie", cookieString);

try
{
Expand Down Expand Up @@ -90,8 +94,11 @@ public static class OpenApiPixiv

httpClient ??= NetworkUtility.GetHttpClient();

if (!httpClient.DefaultRequestHeaders.Contains("Referer"))
httpClient.DefaultRequestHeaders.Add("Referer", "https://www.pixiv.net/");
httpClient.DefaultRequestHeaders.Referrer = new("https://www.pixiv.net/");

var cookieString = instance.Configuration?.GetValue<string>("OpenApi:PixivCookie");
if (!string.IsNullOrWhiteSpace(cookieString))
httpClient.DefaultRequestHeaders.Add("Cookie", cookieString);

try
{
Expand Down Expand Up @@ -121,6 +128,17 @@ public static class OpenApiPixiv

return null;
}
}
}

/// <summary>
/// Returns a proxied Pixiv image URL using the configured proxy template, or the original image URL if no proxy is set.
/// </summary>
/// <param name="instance">OpenApi instance</param>
/// <param name="imageUrl">Original Pixiv image URL</param>
/// <returns>Proxied image URL or original image URL</returns>
public static string GetPixivImageProxyUrl(this OpenApi instance, string imageUrl)
{
var proxyTemplate = instance.Configuration?.GetValue<string>("OpenApi:PixivImageProxy");
return string.IsNullOrWhiteSpace(proxyTemplate) ? imageUrl : string.Format(proxyTemplate, imageUrl);
}
}
}
30 changes: 12 additions & 18 deletions src/RitsukageBotForDiscord/Modules/AI/AiInteractions.Chatting.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,10 +479,9 @@ private async Task<EmbedBuilder[]> ProgressActions(string json)
return [.. result];
}

private async Task<EmbedBuilder?> ProcessingModifyGood(JObject data)
private async Task<EmbedBuilder?> ProcessingModifyGood(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for good action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for good action");
var param = paramToken.ToObject<ActionParam.GoodActionParam>()
?? throw new InvalidDataException("Invalid JSON data for good action");
Expand Down Expand Up @@ -517,10 +516,9 @@ await ChatClientProvider.RecordChatDataChangeHistory(Context.User.Id, "good", pa
return embedBuilder;
}

private async Task<EmbedBuilder?> ProcessingAddShortMemory(JObject data)
private async Task<EmbedBuilder?> ProcessingAddShortMemory(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for add_short_memory action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for add_short_memory action");
var param = paramToken.ToObject<ActionParam.MemoryActionParam>()
?? throw new InvalidDataException("Invalid JSON data for add_short_memory action");
Expand All @@ -546,10 +544,9 @@ await ChatClientProvider
return embed;
}

private async Task<EmbedBuilder?> ProcessingAddLongMemory(JObject data)
private async Task<EmbedBuilder?> ProcessingAddLongMemory(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for add_long_memory action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for add_long_memory action");
var param = paramToken.ToObject<ActionParam.MemoryActionParam>()
?? throw new InvalidDataException("Invalid JSON data for add_long_memory action");
Expand All @@ -575,10 +572,9 @@ await ChatClientProvider
return embed;
}

private async Task<EmbedBuilder?> ProcessingRemoveLongMemory(JObject data)
private async Task<EmbedBuilder?> ProcessingRemoveLongMemory(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for remove_long_memory action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for remove_long_memory action");
var param = paramToken.ToObject<ActionParam.RemoveMemoryActionParam>()
?? throw new InvalidDataException("Invalid JSON data for remove_long_memory action");
Expand All @@ -603,10 +599,9 @@ await ChatClientProvider.RemoveMemory(Context.User.Id, ChatMemoryType.LongTerm,
return embed;
}

private async Task<EmbedBuilder?> ProcessingUpdateSelfState(JObject data)
private async Task<EmbedBuilder?> ProcessingUpdateSelfState(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for update_self_state action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for update_self_state action");
var param = paramToken.ToObject<ActionParam.MemoryActionParam>()
?? throw new InvalidDataException("Invalid JSON data for update_self_state action");
Expand All @@ -631,10 +626,9 @@ await ChatClientProvider
return embed;
}

private async Task<EmbedBuilder?> ProcessingRemoveSelfState(JObject data)
private async Task<EmbedBuilder?> ProcessingRemoveSelfState(JObject? data)
{
if (data is null) throw new InvalidDataException("Invalid JSON data for remove_self_state action");
if (!data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
if (data is null || !data.TryGetValue("param", out var paramValue) || paramValue is not JObject paramToken)
throw new InvalidDataException("Invalid JSON data for remove_self_state action");
var param = paramToken.ToObject<ActionParam.RemoveMemoryActionParam>()
?? throw new InvalidDataException("Invalid JSON data for remove_self_state action");
Expand Down
16 changes: 16 additions & 0 deletions src/RitsukageBotForDiscord/Modules/AI/AiInteractions.Preprocessing.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,22 @@ private async Task<PreprocessingActionData> PreprocessingPixivQueryIllustInfo(JO
sb.AppendLine($"点赞数：{result.LikeCount}");
sb.AppendLine($"评论数：{result.CommentCount}");

if (!string.IsNullOrWhiteSpace(result.Urls.Original))
{
var (success, explainResult) =
await ChatClientProvider.ExplainImageAsync(
OpenApi.Instance.GetPixivImageProxyUrl(result.Urls.Original));
if (success && !string.IsNullOrWhiteSpace(explainResult))
sb.AppendLine($"""
图像内容描述：
```
{explainResult}
```
""");
else
sb.AppendLine($"<图像内容解析失败，原因：{explainResult}>");
}

var otherArtworks = result.UserIllusts?.Where(x => x.Value != null).Select(x =>
{
var illust = x.Value!;
Expand Down
4 changes: 1 addition & 3 deletions src/RitsukageBotForDiscord/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,6 @@
logger.LogInformation("Network default User-Agent: {UserAgent}", UserAgent.Default);

NetworkUtility.SetHttpClientFactory(host.Services.GetRequiredService<IHttpClientFactory>());
OpenApi.Instance.SetCacheProvider(host.Services.GetRequiredService<IFusionCache>());
OpenApi.Instance.SetLogger(host.Services.GetRequiredService<ILogger<OpenApi>>());

OpenApi.Instance.SetServiceProvider(host.Services);

await host.RunAsync(HostCancellationToken.Token).ConfigureAwait(false);
97 changes: 88 additions & 9 deletions src/RitsukageBotForDiscord/Services/Providers/ChatClientProviderService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@ public partial class ChatClientProviderService(IServiceProvider serviceProvider)
private readonly ILogger<ChatClientProviderService> _logger =
serviceProvider.GetRequiredService<ILogger<ChatClientProviderService>>();

private readonly IServiceProvider _serviceProvider = serviceProvider;

private IChatClient CreateChatClient(EndpointConfig config)
{
if (string.IsNullOrWhiteSpace(config.Endpoint))
Expand All @@ -49,7 +47,7 @@ private IChatClient CreateChatClient(EndpointConfig config)
}), config.ModelId);

var client = new ChatClientBuilder(innerChatClient)
.UseDistributedCache(_serviceProvider.GetRequiredService<IDistributedCache>())
.UseDistributedCache(serviceProvider.GetRequiredService<IDistributedCache>())
//.UseLogging(_serviceProvider.GetRequiredService<ILoggerFactory>())
.Build();
return client;
Expand Down Expand Up @@ -84,9 +82,9 @@ public EndpointConfig GetFirstChatEndpoint()
{
var random = _configuration.GetValue<bool>("AI:FirstServiceRandom");
var configs = GetEndpointConfigs();
if (configs.Length == 0)
throw new InvalidDataException("No endpoint is configured");
return configs[random ? Random.Shared.Next(configs.Length) : 0];
return configs.Length == 0
? throw new InvalidDataException("No endpoint is configured")
: configs[random ? Random.Shared.Next(configs.Length) : 0];
}

/// <summary>
Expand All @@ -97,9 +95,9 @@ public EndpointConfig GetFirstChatEndpoint()
public EndpointConfig GetChatEndpointRandomly()
{
var configs = GetEndpointConfigs();
if (configs.Length == 0)
throw new InvalidDataException("No endpoint is configured");
return configs[Random.Shared.Next(configs.Length)];
return configs.Length == 0
? throw new InvalidDataException("No endpoint is configured")
: configs[Random.Shared.Next(configs.Length)];
}

internal EndpointConfig[] GetEndpointConfigs()
Expand Down Expand Up @@ -262,6 +260,79 @@ public string[] GetRoles()
return [.. _configuration.GetSection("AI:RoleData").GetChildren().Select(x => x.Key)];
}

/// <summary>
/// Check if ImageExplainer is enabled
/// </summary>
/// <returns></returns>
public bool IsImageExplainerEnabled()
{
var config = _configuration.GetSection("AI:Function:ImageExplainer").Get<ImageExplainerConfig>();
return config?.Enabled ?? false;
}

/// <summary>
/// Get ImageExplainer configuration
/// </summary>
/// <returns></returns>
public ImageExplainerConfig? GetImageExplainerConfig()
{
return _configuration.GetSection("AI:Function:ImageExplainer").Get<ImageExplainerConfig>();
}

/// <summary>
/// Explain image content using AI
/// </summary>
/// <param name="imageUrl">The URL of the image to explain</param>
/// <returns>The explanation of the image content</returns>
/// <exception cref="InvalidOperationException">Thrown when ImageExplainer is not enabled or configured properly</exception>
public async Task<(bool, string)> ExplainImageAsync(string imageUrl)
{
var config = GetImageExplainerConfig();
if (config is null || !config.Enabled)
return (false, "ImageExplainer is not enabled or configured");

if (string.IsNullOrWhiteSpace(imageUrl))
return (false, "Image URL cannot be null or empty");

try
{
var chatClient = CreateChatClient(config.Service);

// Get prompt from configuration
if (!GetPrompt(config.PromptConfig, out var prompt))
{
_logger.LogWarning("ImageExplainer prompt is empty, using default");
prompt =
"Please describe the content of this image in detail, including the main object, scene, color, composition, etc.";
}

var messages = new List<ChatMessage>
{
new(ChatRole.System,
"You are a professional image analysis assistant who can describe image content in detail."),
new(ChatRole.User, [
new TextContent(prompt),
new ImageContent(imageUrl),
]),
};

var chatOptions = new ChatOptions
{
Temperature = config.PromptConfig.Temperature,
};

var response = await chatClient.CompleteAsync(messages, chatOptions).ConfigureAwait(false);
if (!string.IsNullOrWhiteSpace(response.Message.Text))
return (true, response.Message.Text);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to explain image: {ImageUrl}", imageUrl);
}

return (false, "Failed to explain image");
}

/// <summary>
/// Get memory
/// </summary>
Expand Down Expand Up @@ -686,6 +757,14 @@ public record PromptConfig(string Prompt = "", string PromptFile = "", float Tem
/// <param name="Service"></param>
/// <param name="PromptConfig"></param>
public record AssistantConfig(bool Enabled, EndpointConfig Service, PromptConfig PromptConfig);

/// <summary>
/// Image explainer configuration
/// </summary>
/// <param name="Enabled"></param>
/// <param name="Service"></param>
/// <param name="PromptConfig"></param>
public record ImageExplainerConfig(bool Enabled, EndpointConfig Service, PromptConfig PromptConfig);
}

/// <summary>
Expand Down
18 changes: 18 additions & 0 deletions src/RitsukageBotForDiscord/appsettings.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@
}
]
},
"OpenApi": {
"PixivImageProxy": "{0}",
"PixivCookie": ""
},
"AI": {
"Enabled": false,
"Service": [
Expand Down Expand Up @@ -87,6 +91,20 @@
"TimeBroadcast": {
"Enabled": false,
"Prompt": ""
},
"ImageExplainer": {
"Enabled": false,
"Service": {
"Name": "",
"Endpoint": "",
"ApiKey": "",
"ModelId": ""
},
"PromptConfig": {
"Prompt": "Please describe the content of this image in detail, including the main object, scene, color, composition, etc.",
"PromptFile": "",
"Temperature": 0.7
}
}
},
"Config": {
Expand Down
Loading