Add subtitle translation presets, custom instructions, and per-series glossary #2
No files matched your search
@@ -8,6 +8,12 @@ ecm_add_test(mpvuserconfigtest.cpp
|
|||||||
LINK_LIBRARIES qtpotato_core Qt6::Test
|
LINK_LIBRARIES qtpotato_core Qt6::Test
|
||||||
)
|
)
|
||||||
|
|
||||||
|
ecm_add_test(glossarystoretest.cpp ../src/translate/glossarystore.cpp
|
||||||
|
TEST_NAME glossarystoretest
|
||||||
|
LINK_LIBRARIES Qt6::Core Qt6::Test
|
||||||
|
)
|
||||||
|
target_include_directories(glossarystoretest PRIVATE ../src/translate)
|
||||||
|
|
||||||
ecm_add_test(playlistmodeltest.cpp ../src/playlist/playlistmodel.cpp
|
ecm_add_test(playlistmodeltest.cpp ../src/playlist/playlistmodel.cpp
|
||||||
TEST_NAME playlistmodeltest
|
TEST_NAME playlistmodeltest
|
||||||
LINK_LIBRARIES Qt6::Qml Qt6::Test
|
LINK_LIBRARIES Qt6::Qml Qt6::Test
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
#include "glossarystore.h"
|
||||||
|
|
||||||
|
#include <QStandardPaths>
|
||||||
|
#include <QTest>
|
||||||
|
using namespace Qt::Literals::StringLiterals;
|
||||||
|
|
||||||
|
class GlossaryStoreTest : public QObject
|
||||||
|
{
|
||||||
|
Q_OBJECT
|
||||||
|
private Q_SLOTS:
|
||||||
|
void saveLoadMerge()
|
||||||
|
{
|
||||||
|
QTemporaryDir temp;
|
||||||
|
QStandardPaths::setTestModeEnabled(true);
|
||||||
|
const QString directory = temp.path() + u"/series"_s;
|
||||||
|
GlossaryStore::Glossary original{directory, {{u"Lin"_s, u"Lâm"_s}}};
|
||||||
|
QVERIFY(GlossaryStore::save(original));
|
||||||
|
const auto loaded = GlossaryStore::load(directory);
|
||||||
|
QCOMPARE(loaded.entries.size(), 1);
|
||||||
|
const auto merged = GlossaryStore::merge(loaded, {{u"Lin"_s, u"Other"_s}, {u"Sect"_s, u"Tông"_s}});
|
||||||
|
QCOMPARE(merged.entries.size(), 2);
|
||||||
|
QCOMPARE(merged.entries.at(0).target, u"Lâm"_s);
|
||||||
|
QCOMPARE(merged.entries.at(1).source, u"Sect"_s);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
QTEST_MAIN(GlossaryStoreTest)
|
||||||
|
#include "glossarystoretest.moc"
|
||||||
@@ -34,6 +34,8 @@ add_executable(qtpotato
|
|||||||
playlist/playlistmodel.h
|
playlist/playlistmodel.h
|
||||||
translate/apikeystore.cpp
|
translate/apikeystore.cpp
|
||||||
translate/apikeystore.h
|
translate/apikeystore.h
|
||||||
|
translate/glossarystore.cpp
|
||||||
|
translate/glossarystore.h
|
||||||
translate/llmclient.cpp
|
translate/llmclient.cpp
|
||||||
translate/llmclient.h
|
translate/llmclient.h
|
||||||
translate/translationservice.cpp
|
translate/translationservice.cpp
|
||||||
|
|||||||
@@ -35,6 +35,7 @@ Item {
|
|||||||
signal headerToggleRequested()
|
signal headerToggleRequested()
|
||||||
signal contextMenuRequested()
|
signal contextMenuRequested()
|
||||||
signal translateRequested()
|
signal translateRequested()
|
||||||
|
signal editGlossaryRequested()
|
||||||
|
|
||||||
function seekBy(seconds: real) {
|
function seekBy(seconds: real) {
|
||||||
mpv.seekBy(seconds)
|
mpv.seekBy(seconds)
|
||||||
@@ -364,6 +365,13 @@ Item {
|
|||||||
onTriggered: root.translateRequested()
|
onTriggered: root.translateRequested()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
property Action editGlossary: Action {
|
||||||
|
enabled: root.shortcutsEnabled && root.hasFile && root.mpv.currentUrl.toString().startsWith("file:")
|
||||||
|
text: KI18n.i18nc("@action", "Edit Translation Glossary…")
|
||||||
|
icon.name: "document-edit"
|
||||||
|
onTriggered: root.editGlossaryRequested()
|
||||||
|
}
|
||||||
|
|
||||||
property Action cancelTranslation: Action {
|
property Action cancelTranslation: Action {
|
||||||
enabled: root.translator.busy
|
enabled: root.translator.busy
|
||||||
text: KI18n.i18nc("@action", "Cancel Translation")
|
text: KI18n.i18nc("@action", "Cancel Translation")
|
||||||
|
|||||||
@@ -5,6 +5,7 @@
|
|||||||
import QtQuick
|
import QtQuick
|
||||||
import QtQuick.Controls
|
import QtQuick.Controls
|
||||||
import QtQuick.Dialogs
|
import QtQuick.Dialogs
|
||||||
|
import QtQuick.Layouts
|
||||||
import org.kde.ki18n
|
import org.kde.ki18n
|
||||||
import org.kde.kirigami as Kirigami
|
import org.kde.kirigami as Kirigami
|
||||||
import org.kde.config as KConfig
|
import org.kde.config as KConfig
|
||||||
@@ -108,6 +109,10 @@ ApplicationWindow {
|
|||||||
onPlaylistToggleRequested: playlistPanel.toggle()
|
onPlaylistToggleRequested: playlistPanel.toggle()
|
||||||
onContextMenuRequested: contextMenu.popup()
|
onContextMenuRequested: contextMenu.popup()
|
||||||
onTranslateRequested: appWindow.translateCurrentSubtitle()
|
onTranslateRequested: appWindow.translateCurrentSubtitle()
|
||||||
|
onEditGlossaryRequested: {
|
||||||
|
translationService.loadGlossary(video.currentUrl)
|
||||||
|
glossaryDialog.open()
|
||||||
|
}
|
||||||
onMenuBarToggleRequested: {
|
onMenuBarToggleRequested: {
|
||||||
QtPotatoSettings.showMenuBar = !QtPotatoSettings.showMenuBar
|
QtPotatoSettings.showMenuBar = !QtPotatoSettings.showMenuBar
|
||||||
QtPotatoSettings.save()
|
QtPotatoSettings.save()
|
||||||
@@ -312,4 +317,40 @@ ApplicationWindow {
|
|||||||
]
|
]
|
||||||
onAccepted: video.addSubtitleFile(selectedFile)
|
onAccepted: video.addSubtitleFile(selectedFile)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Kirigami.Dialog {
|
||||||
|
id: glossaryDialog
|
||||||
|
title: KI18n.i18nc("@title:window", "Translation Glossary")
|
||||||
|
standardButtons: Kirigami.Dialog.Save | Kirigami.Dialog.Cancel
|
||||||
|
width: Math.min(appWindow.width * 0.9, 650)
|
||||||
|
contentItem: ColumnLayout {
|
||||||
|
ListView {
|
||||||
|
id: glossaryList
|
||||||
|
Layout.fillWidth: true
|
||||||
|
Layout.fillHeight: true
|
||||||
|
Layout.preferredHeight: Kirigami.Units.gridUnit * 15
|
||||||
|
clip: true
|
||||||
|
model: ListModel { id: glossaryModel }
|
||||||
|
delegate: RowLayout {
|
||||||
|
width: glossaryList.width
|
||||||
|
TextField { text: source; Layout.fillWidth: true; onTextChanged: model.source = text }
|
||||||
|
TextField { text: target; Layout.fillWidth: true; onTextChanged: model.target = text }
|
||||||
|
ToolButton { icon.name: "edit-delete-remove"; onClicked: glossaryModel.remove(index) }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
RowLayout {
|
||||||
|
Button { text: KI18n.i18nc("@action:button", "Add"); onClicked: glossaryModel.append({source: "", target: ""}) }
|
||||||
|
Button { text: KI18n.i18nc("@action:button", "Clear"); onClicked: glossaryModel.clear() }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
onOpened: {
|
||||||
|
glossaryModel.clear()
|
||||||
|
for (const entry of translationService.glossaryEntries) glossaryModel.append(entry)
|
||||||
|
}
|
||||||
|
onAccepted: {
|
||||||
|
const entries = []
|
||||||
|
for (let i = 0; i < glossaryModel.count; ++i) entries.push(glossaryModel.get(i))
|
||||||
|
translationService.saveGlossary(entries)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -26,6 +26,7 @@ Menu {
|
|||||||
}
|
}
|
||||||
MenuSeparator {}
|
MenuSeparator {}
|
||||||
MenuItem { action: root.actions.translateSubtitle }
|
MenuItem { action: root.actions.translateSubtitle }
|
||||||
|
MenuItem { action: root.actions.editGlossary }
|
||||||
MenuItem {
|
MenuItem {
|
||||||
action: root.actions.cancelTranslation
|
action: root.actions.cancelTranslation
|
||||||
visible: root.actions.cancelTranslation.enabled
|
visible: root.actions.cancelTranslation.enabled
|
||||||
|
|||||||
@@ -190,6 +190,13 @@ SettingsBasePage {
|
|||||||
onEditingFinished: QtPotatoSettings.save()
|
onEditingFinished: QtPotatoSettings.save()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CheckBox {
|
||||||
|
text: KI18n.i18nc("@option:check", "Build a glossary automatically before translating")
|
||||||
|
checked: QtPotatoSettings.autoBuildGlossary
|
||||||
|
Layout.columnSpan: 2
|
||||||
|
onToggled: { QtPotatoSettings.autoBuildGlossary = checked; QtPotatoSettings.save() }
|
||||||
|
}
|
||||||
|
|
||||||
Label {
|
Label {
|
||||||
text: KI18n.i18nc("@label:spinbox", "Lines per request")
|
text: KI18n.i18nc("@label:spinbox", "Lines per request")
|
||||||
Layout.alignment: Qt.AlignRight
|
Layout.alignment: Qt.AlignRight
|
||||||
|
|||||||
@@ -130,6 +130,10 @@
|
|||||||
<label>Additional instructions for subtitle translation</label>
|
<label>Additional instructions for subtitle translation</label>
|
||||||
<default></default>
|
<default></default>
|
||||||
</entry>
|
</entry>
|
||||||
|
<entry name="AutoBuildGlossary" type="Bool">
|
||||||
|
<label>Automatically build a glossary before translating</label>
|
||||||
|
<default>true</default>
|
||||||
|
</entry>
|
||||||
<entry name="BatchSize" type="Int">
|
<entry name="BatchSize" type="Int">
|
||||||
<label>Subtitle lines sent per request</label>
|
<label>Subtitle lines sent per request</label>
|
||||||
<default>40</default>
|
<default>40</default>
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// SPDX-FileCopyrightText: 2026 Tri Vo Khac
|
||||||
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
#include "glossarystore.h"
|
||||||
|
|
||||||
|
#include <QCryptographicHash>
|
||||||
|
#include <QDir>
|
||||||
|
#include <QFile>
|
||||||
|
#include <QFileInfo>
|
||||||
|
#include <QJsonArray>
|
||||||
|
#include <QJsonDocument>
|
||||||
|
#include <QJsonObject>
|
||||||
|
#include <QSaveFile>
|
||||||
|
#include <QSet>
|
||||||
|
#include <QStandardPaths>
|
||||||
|
|
||||||
|
using namespace Qt::Literals::StringLiterals;
|
||||||
|
|
||||||
|
QString GlossaryStore::pathForDirectory(const QString &directory)
|
||||||
|
{
|
||||||
|
const QString root = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation) + u"/glossaries"_s;
|
||||||
|
const QByteArray digest = QCryptographicHash::hash(QFile::encodeName(QDir::cleanPath(directory)), QCryptographicHash::Sha1).toHex();
|
||||||
|
return root + u"/"_s + QString::fromLatin1(digest) + u".json"_s;
|
||||||
|
}
|
||||||
|
|
||||||
|
static QJsonObject glossaryJson(const GlossaryStore::Glossary &glossary)
|
||||||
|
{
|
||||||
|
QJsonArray entries;
|
||||||
|
for (const auto &entry : glossary.entries) {
|
||||||
|
if (!entry.source.trimmed().isEmpty() && !entry.target.trimmed().isEmpty()) {
|
||||||
|
entries.append(QJsonObject{{u"source"_s, entry.source}, {u"target"_s, entry.target}});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return QJsonObject{{u"directory"_s, glossary.directory}, {u"entries"_s, entries}};
|
||||||
|
}
|
||||||
|
|
||||||
|
GlossaryStore::Glossary GlossaryStore::load(const QString &directory)
|
||||||
|
{
|
||||||
|
Glossary result{QDir::cleanPath(directory), {}};
|
||||||
|
QFile file(pathForDirectory(result.directory));
|
||||||
|
if (!file.open(QIODevice::ReadOnly)) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
const QJsonObject object = QJsonDocument::fromJson(file.readAll()).object();
|
||||||
|
for (const QJsonValue &value : object.value(u"entries"_s).toArray()) {
|
||||||
|
const QJsonObject entry = value.toObject();
|
||||||
|
const QString source = entry.value(u"source"_s).toString().trimmed();
|
||||||
|
const QString target = entry.value(u"target"_s).toString().trimmed();
|
||||||
|
if (!source.isEmpty() && !target.isEmpty()) {
|
||||||
|
result.entries.append({source, target});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool GlossaryStore::save(const Glossary &glossary, QString *error)
|
||||||
|
{
|
||||||
|
const QString directory = QDir::cleanPath(glossary.directory);
|
||||||
|
const QString path = pathForDirectory(directory);
|
||||||
|
QDir().mkpath(QFileInfo(path).absolutePath());
|
||||||
|
QSaveFile file(path);
|
||||||
|
if (!file.open(QIODevice::WriteOnly) || file.write(QJsonDocument(glossaryJson({directory, glossary.entries})).toJson(QJsonDocument::Indented)) < 0 || !file.commit()) {
|
||||||
|
if (error) {
|
||||||
|
*error = file.errorString();
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
GlossaryStore::Glossary GlossaryStore::merge(const Glossary &existing, const QList<Entry> &additions)
|
||||||
|
{
|
||||||
|
Glossary result{QDir::cleanPath(existing.directory), existing.entries};
|
||||||
|
QSet<QString> sources;
|
||||||
|
for (const Entry &entry : result.entries) {
|
||||||
|
sources.insert(entry.source.trimmed());
|
||||||
|
}
|
||||||
|
for (const Entry &entry : additions) {
|
||||||
|
const QString source = entry.source.trimmed();
|
||||||
|
const QString target = entry.target.trimmed();
|
||||||
|
if (!source.isEmpty() && !target.isEmpty() && !sources.contains(source)) {
|
||||||
|
result.entries.append({source, target});
|
||||||
|
sources.insert(source);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
QString GlossaryStore::serialize(const Glossary &glossary)
|
||||||
|
{
|
||||||
|
return QString::fromUtf8(QJsonDocument(glossaryJson(glossary)).toJson(QJsonDocument::Compact));
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
// SPDX-FileCopyrightText: 2026 Tri Vo Khac
|
||||||
|
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <QList>
|
||||||
|
#include <QString>
|
||||||
|
|
||||||
|
class GlossaryStore
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
struct Entry { QString source; QString target; };
|
||||||
|
struct Glossary { QString directory; QList<Entry> entries; };
|
||||||
|
|
||||||
|
static Glossary load(const QString &directory);
|
||||||
|
static bool save(const Glossary &glossary, QString *error = nullptr);
|
||||||
|
static Glossary merge(const Glossary &existing, const QList<Entry> &additions);
|
||||||
|
static QString serialize(const Glossary &glossary);
|
||||||
|
static QString pathForDirectory(const QString &directory);
|
||||||
|
};
|
||||||
@@ -30,7 +30,7 @@ struct ReplyDeleter {
|
|||||||
};
|
};
|
||||||
using ReplyPtr = std::unique_ptr<QNetworkReply, ReplyDeleter>;
|
using ReplyPtr = std::unique_ptr<QNetworkReply, ReplyDeleter>;
|
||||||
|
|
||||||
QString systemPrompt(const QString &language, const QString &presetId, const QString &customInstructions)
|
QString systemPrompt(const QString &language, const QString &presetId, const QString &glossaryText, const QString &customInstructions)
|
||||||
{
|
{
|
||||||
const auto &preset = TranslationPresets::find(presetId);
|
const auto &preset = TranslationPresets::find(presetId);
|
||||||
QString prompt = uR"(You are a professional subtitle translator. Translate subtitles into %1.
|
QString prompt = uR"(You are a professional subtitle translator. Translate subtitles into %1.
|
||||||
@@ -47,6 +47,10 @@ Style guidance:
|
|||||||
|
|
||||||
)"_s
|
)"_s
|
||||||
.arg(language, preset.guidance);
|
.arg(language, preset.guidance);
|
||||||
|
if (!glossaryText.trimmed().isEmpty()) {
|
||||||
|
prompt += u"Glossary — always use these renderings exactly:\n"_s;
|
||||||
|
prompt += glossaryText.trimmed() + u"\n\n"_s;
|
||||||
|
}
|
||||||
if (!customInstructions.trimmed().isEmpty()) {
|
if (!customInstructions.trimmed().isEmpty()) {
|
||||||
prompt += u"Additional instructions from the user (follow them unless they conflict with the reply format):\n"_s;
|
prompt += u"Additional instructions from the user (follow them unless they conflict with the reply format):\n"_s;
|
||||||
prompt += customInstructions.trimmed();
|
prompt += customInstructions.trimmed();
|
||||||
@@ -149,7 +153,7 @@ QCoro::Task<LlmClient::BatchResult> LlmClient::translate(QStringList lines, QStr
|
|||||||
{"model"_L1, m_config.model},
|
{"model"_L1, m_config.model},
|
||||||
{"messages"_L1,
|
{"messages"_L1,
|
||||||
QJsonArray{
|
QJsonArray{
|
||||||
QJsonObject{{"role"_L1, "system"_L1}, {"content"_L1, systemPrompt(m_config.targetLanguage, m_config.translationPreset, m_config.customInstructions)}},
|
QJsonObject{{"role"_L1, "system"_L1}, {"content"_L1, systemPrompt(m_config.targetLanguage, m_config.translationPreset, m_config.glossaryText, m_config.customInstructions)}},
|
||||||
QJsonObject{{"role"_L1, "user"_L1}, {"content"_L1, QString::fromUtf8(QJsonDocument(userContent).toJson(QJsonDocument::Compact))}},
|
QJsonObject{{"role"_L1, "user"_L1}, {"content"_L1, QString::fromUtf8(QJsonDocument(userContent).toJson(QJsonDocument::Compact))}},
|
||||||
}},
|
}},
|
||||||
};
|
};
|
||||||
@@ -192,6 +196,39 @@ QCoro::Task<LlmClient::BatchResult> LlmClient::translate(QStringList lines, QStr
|
|||||||
co_return result;
|
co_return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
QCoro::Task<LlmClient::BatchResult> LlmClient::extractGlossary(QStringList lines, QString mediaTitle)
|
||||||
|
{
|
||||||
|
const auto &preset = TranslationPresets::find(m_config.translationPreset);
|
||||||
|
const QString prompt = u"Identify recurring proper nouns and special terms in these subtitles. Return only JSON of the form {\"entries\":[{\"source\":\"...\",\"target\":\"...\"}]}. Include people, places, sects/clans, techniques, ranks and recurring special terms; omit ordinary words. Follow the target language and style preset.\nTarget language: "_s + m_config.targetLanguage + u"\nStyle guidance:\n"_s + preset.guidance;
|
||||||
|
const QJsonObject userContent{{"title"_L1, mediaTitle}, {"lines"_L1, QJsonArray::fromStringList(lines)}};
|
||||||
|
const QJsonObject body{{"model"_L1, m_config.model}, {"messages"_L1, QJsonArray{
|
||||||
|
QJsonObject{{"role"_L1, "system"_L1}, {"content"_L1, prompt}},
|
||||||
|
QJsonObject{{"role"_L1, "user"_L1}, {"content"_L1, QString::fromUtf8(QJsonDocument(userContent).toJson(QJsonDocument::Compact))}},
|
||||||
|
}}};
|
||||||
|
QNetworkRequest request(QUrl(m_config.baseUrl.toString() + u"/chat/completions"_s));
|
||||||
|
request.setHeader(QNetworkRequest::ContentTypeHeader, u"application/json"_s);
|
||||||
|
if (!m_config.apiKey.isEmpty()) request.setRawHeader("Authorization", "Bearer " + m_config.apiKey.toUtf8());
|
||||||
|
request.setTransferTimeout(RequestTimeoutMs);
|
||||||
|
ReplyPtr reply(m_network->post(request, QJsonDocument(body).toJson(QJsonDocument::Compact)));
|
||||||
|
m_replies.append(reply.get());
|
||||||
|
co_await reply.get();
|
||||||
|
m_replies.removeAll(reply.get());
|
||||||
|
const QByteArray responseBody = reply->readAll();
|
||||||
|
if (reply->error() != QNetworkReply::NoError) co_return BatchResult{{}, describeHttpError(reply.get(), responseBody), isRetryable(reply.get())};
|
||||||
|
const QJsonObject response = QJsonDocument::fromJson(responseBody).object();
|
||||||
|
const QString content = response.value("choices"_L1).toArray().at(0).toObject().value("message"_L1).toObject().value("content"_L1).toString();
|
||||||
|
const QJsonObject object = extractJsonObject(content);
|
||||||
|
if (!object.contains("entries"_L1)) co_return BatchResult{{}, i18n("The model did not return a glossary: %1", content.left(300).simplified()), true};
|
||||||
|
QStringList encoded;
|
||||||
|
for (const QJsonValue &value : object.value("entries"_L1).toArray()) {
|
||||||
|
const QJsonObject entry = value.toObject();
|
||||||
|
const QString source = entry.value("source"_L1).toString().trimmed();
|
||||||
|
const QString target = entry.value("target"_L1).toString().trimmed();
|
||||||
|
if (!source.isEmpty() && !target.isEmpty()) encoded.append(source + u"\t"_s + target);
|
||||||
|
}
|
||||||
|
co_return BatchResult{encoded, {}, false};
|
||||||
|
}
|
||||||
|
|
||||||
QCoro::Task<LlmClient::ModelsResult> LlmClient::listModels()
|
QCoro::Task<LlmClient::ModelsResult> LlmClient::listModels()
|
||||||
{
|
{
|
||||||
QNetworkRequest request(QUrl(m_config.baseUrl.toString() + u"/models"_s));
|
QNetworkRequest request(QUrl(m_config.baseUrl.toString() + u"/models"_s));
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ public:
|
|||||||
QString targetLanguage;
|
QString targetLanguage;
|
||||||
QString translationPreset;
|
QString translationPreset;
|
||||||
QString customInstructions;
|
QString customInstructions;
|
||||||
|
QString glossaryText;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct BatchResult {
|
struct BatchResult {
|
||||||
@@ -41,6 +42,7 @@ public:
|
|||||||
// Translates \a lines. \a context holds the source lines right before them, so the
|
// Translates \a lines. \a context holds the source lines right before them, so the
|
||||||
// model can follow the conversation; they are not translated.
|
// model can follow the conversation; they are not translated.
|
||||||
QCoro::Task<BatchResult> translate(QStringList lines, QStringList context, QString mediaTitle);
|
QCoro::Task<BatchResult> translate(QStringList lines, QStringList context, QString mediaTitle);
|
||||||
|
QCoro::Task<BatchResult> extractGlossary(QStringList lines, QString mediaTitle);
|
||||||
QCoro::Task<ModelsResult> listModels();
|
QCoro::Task<ModelsResult> listModels();
|
||||||
|
|
||||||
// Aborts every request in flight.
|
// Aborts every request in flight.
|
||||||
|
|||||||
@@ -127,6 +127,48 @@ bool TranslationService::isFetchingModels() const
|
|||||||
return m_fetchingModels;
|
return m_fetchingModels;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
QVariantList TranslationService::glossaryEntries() const
|
||||||
|
{
|
||||||
|
QVariantList result;
|
||||||
|
for (const auto &entry : m_glossary.entries) result.append(QVariantMap{{u"source"_s, entry.source}, {u"target"_s, entry.target}});
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
QString TranslationService::glossaryDirectory() const
|
||||||
|
{
|
||||||
|
return m_glossary.directory;
|
||||||
|
}
|
||||||
|
|
||||||
|
void TranslationService::loadGlossary(const QUrl &videoUrl)
|
||||||
|
{
|
||||||
|
const QString directory = videoUrl.isLocalFile() ? QFileInfo(videoUrl.toLocalFile()).absolutePath() : QString();
|
||||||
|
m_glossary = directory.isEmpty() ? GlossaryStore::Glossary{} : GlossaryStore::load(directory);
|
||||||
|
Q_EMIT glossaryChanged();
|
||||||
|
}
|
||||||
|
|
||||||
|
void TranslationService::saveGlossary(const QVariantList &entries)
|
||||||
|
{
|
||||||
|
QList<GlossaryStore::Entry> result;
|
||||||
|
for (const QVariant &value : entries) {
|
||||||
|
const QVariantMap map = value.toMap();
|
||||||
|
result.append({map.value(u"source"_s).toString().trimmed(), map.value(u"target"_s).toString().trimmed()});
|
||||||
|
}
|
||||||
|
m_glossary.entries = result;
|
||||||
|
if (!m_glossary.directory.isEmpty()) {
|
||||||
|
GlossaryStore::save(m_glossary);
|
||||||
|
}
|
||||||
|
Q_EMIT glossaryChanged();
|
||||||
|
}
|
||||||
|
|
||||||
|
void TranslationService::clearGlossary()
|
||||||
|
{
|
||||||
|
m_glossary.entries.clear();
|
||||||
|
if (!m_glossary.directory.isEmpty()) {
|
||||||
|
GlossaryStore::save(m_glossary);
|
||||||
|
}
|
||||||
|
Q_EMIT glossaryChanged();
|
||||||
|
}
|
||||||
|
|
||||||
void TranslationService::translate(const QUrl &videoUrl, const QVariantMap &track, const QString &mediaTitle)
|
void TranslationService::translate(const QUrl &videoUrl, const QVariantMap &track, const QString &mediaTitle)
|
||||||
{
|
{
|
||||||
cancel();
|
cancel();
|
||||||
@@ -172,6 +214,7 @@ QCoro::Task<std::pair<LlmClient::Config, QString>> TranslationService::loadConfi
|
|||||||
settings->targetLanguage().trimmed(),
|
settings->targetLanguage().trimmed(),
|
||||||
settings->translationPreset().trimmed(),
|
settings->translationPreset().trimmed(),
|
||||||
settings->customInstructions(),
|
settings->customInstructions(),
|
||||||
|
{},
|
||||||
};
|
};
|
||||||
|
|
||||||
if (!config.baseUrl.isValid() || config.baseUrl.scheme().isEmpty() || config.baseUrl.host().isEmpty()) {
|
if (!config.baseUrl.isValid() || config.baseUrl.scheme().isEmpty() || config.baseUrl.host().isEmpty()) {
|
||||||
@@ -220,7 +263,7 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
|
|||||||
setBusy(true);
|
setBusy(true);
|
||||||
setStatusText(i18n("Reading subtitles…"));
|
setStatusText(i18n("Reading subtitles…"));
|
||||||
|
|
||||||
const auto [config, configError] = co_await loadConfig();
|
auto [config, configError] = co_await loadConfig();
|
||||||
if (job->cancelled) {
|
if (job->cancelled) {
|
||||||
co_return;
|
co_return;
|
||||||
}
|
}
|
||||||
@@ -247,15 +290,25 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
|
|||||||
co_return;
|
co_return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Same subtitles, model, language, and prompt settings give the same file, so it is translated once.
|
// Episodes in one folder share a glossary. Streams have no folder, so they get none.
|
||||||
|
const QString seriesDirectory = videoUrl.isLocalFile() ? QFileInfo(videoUrl.toLocalFile()).absolutePath() : QString();
|
||||||
|
const bool useGlossary = !seriesDirectory.isEmpty();
|
||||||
|
GlossaryStore::Glossary glossary = useGlossary ? GlossaryStore::load(seriesDirectory) : GlossaryStore::Glossary{};
|
||||||
|
const QString sourceSrt = Subtitles::toSrt(cues);
|
||||||
|
|
||||||
|
// Same subtitles, model, language, prompt settings and glossary give the same file, so it is translated once.
|
||||||
|
const auto outputPathFor = [&](const GlossaryStore::Glossary &g) {
|
||||||
QCryptographicHash hash(QCryptographicHash::Sha1);
|
QCryptographicHash hash(QCryptographicHash::Sha1);
|
||||||
hash.addData(CacheVersion);
|
hash.addData(CacheVersion);
|
||||||
hash.addData(config.model.toUtf8());
|
hash.addData(config.model.toUtf8());
|
||||||
hash.addData(config.targetLanguage.toUtf8());
|
hash.addData(config.targetLanguage.toUtf8());
|
||||||
hash.addData(config.translationPreset.toUtf8());
|
hash.addData(config.translationPreset.toUtf8());
|
||||||
hash.addData(config.customInstructions.toUtf8());
|
hash.addData(config.customInstructions.toUtf8());
|
||||||
hash.addData(Subtitles::toSrt(cues).toUtf8());
|
hash.addData(GlossaryStore::serialize(g).toUtf8());
|
||||||
const QString outputPath = cachePath(videoUrl, QString::fromLatin1(hash.result().toHex().left(12)), config.targetLanguage);
|
hash.addData(sourceSrt.toUtf8());
|
||||||
|
return cachePath(videoUrl, QString::fromLatin1(hash.result().toHex().left(12)), config.targetLanguage);
|
||||||
|
};
|
||||||
|
QString outputPath = outputPathFor(glossary);
|
||||||
|
|
||||||
if (QFileInfo::exists(outputPath)) {
|
if (QFileInfo::exists(outputPath)) {
|
||||||
m_job.reset();
|
m_job.reset();
|
||||||
@@ -265,6 +318,55 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
|
|||||||
co_return;
|
co_return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (useGlossary && QtPotatoSettings::self()->autoBuildGlossary()) {
|
||||||
|
setStatusText(i18n("Building glossary…"));
|
||||||
|
QStringList extractionLines;
|
||||||
|
QHash<QString, bool> seen;
|
||||||
|
for (const SubtitleCue &cue : cues) {
|
||||||
|
if (!seen.contains(cue.text)) { seen.insert(cue.text, true); extractionLines.append(cue.text); }
|
||||||
|
}
|
||||||
|
constexpr qsizetype MaxGlossaryChars = 30000;
|
||||||
|
// Long subtitles are sampled evenly so names from the whole episode are seen.
|
||||||
|
qsizetype allChars = 0;
|
||||||
|
for (const QString &line : std::as_const(extractionLines)) {
|
||||||
|
allChars += line.size() + 1;
|
||||||
|
}
|
||||||
|
const qsizetype step = std::max<qsizetype>(1, (allChars + MaxGlossaryChars - 1) / MaxGlossaryChars);
|
||||||
|
QStringList sampled;
|
||||||
|
qsizetype total = 0;
|
||||||
|
for (qsizetype i = 0; i < extractionLines.size(); i += step) {
|
||||||
|
if (total + extractionLines.at(i).size() + 1 > MaxGlossaryChars) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
sampled.append(extractionLines.at(i));
|
||||||
|
total += extractionLines.at(i).size() + 1;
|
||||||
|
}
|
||||||
|
job->client = std::make_unique<LlmClient>(m_network, config);
|
||||||
|
const LlmClient::BatchResult extracted = co_await job->client->extractGlossary(sampled, job->mediaTitle);
|
||||||
|
if (job->cancelled) {
|
||||||
|
co_return;
|
||||||
|
}
|
||||||
|
if (extracted.error.isEmpty()) {
|
||||||
|
QList<GlossaryStore::Entry> additions;
|
||||||
|
for (const QString &encoded : extracted.translations) {
|
||||||
|
const qsizetype tab = encoded.indexOf(u'\t');
|
||||||
|
if (tab > 0) additions.append({encoded.left(tab), encoded.mid(tab + 1)});
|
||||||
|
}
|
||||||
|
glossary = GlossaryStore::merge(glossary, additions);
|
||||||
|
GlossaryStore::save(glossary);
|
||||||
|
} else {
|
||||||
|
setStatusText(i18n("Glossary unavailable; translating without it."));
|
||||||
|
}
|
||||||
|
outputPath = outputPathFor(glossary);
|
||||||
|
if (QFileInfo::exists(outputPath)) {
|
||||||
|
m_job.reset();
|
||||||
|
setBusy(false);
|
||||||
|
setStatusText({});
|
||||||
|
Q_EMIT translated(QUrl::fromLocalFile(outputPath), true);
|
||||||
|
co_return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Repeated lines (songs, catchphrases) are sent once.
|
// Repeated lines (songs, catchphrases) are sent once.
|
||||||
QStringList uniqueLines;
|
QStringList uniqueLines;
|
||||||
QHash<QString, qsizetype> uniqueIndex;
|
QHash<QString, qsizetype> uniqueIndex;
|
||||||
@@ -278,6 +380,12 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
|
|||||||
cueToUnique.append(*it);
|
cueToUnique.append(*it);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
QStringList glossaryLines;
|
||||||
|
for (const auto &entry : std::as_const(glossary.entries)) {
|
||||||
|
glossaryLines.append(entry.source + u" → "_s + entry.target);
|
||||||
|
}
|
||||||
|
config.glossaryText = glossaryLines.join(u'\n');
|
||||||
|
|
||||||
job->client = std::make_unique<LlmClient>(m_network, config);
|
job->client = std::make_unique<LlmClient>(m_network, config);
|
||||||
auto batches = std::make_shared<Batches>();
|
auto batches = std::make_shared<Batches>();
|
||||||
batches->lines = uniqueLines;
|
batches->lines = uniqueLines;
|
||||||
|
|||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
#include "llmclient.h"
|
#include "llmclient.h"
|
||||||
#include "subtitleextractor.h"
|
#include "subtitleextractor.h"
|
||||||
|
#include "glossarystore.h"
|
||||||
|
|
||||||
#include <QCoroTask>
|
#include <QCoroTask>
|
||||||
|
|
||||||
@@ -31,6 +32,8 @@ class TranslationService : public QObject
|
|||||||
Q_PROPERTY(QStringList availableModels READ availableModels NOTIFY availableModelsChanged)
|
Q_PROPERTY(QStringList availableModels READ availableModels NOTIFY availableModelsChanged)
|
||||||
Q_PROPERTY(QVariantList translationPresets READ translationPresets CONSTANT)
|
Q_PROPERTY(QVariantList translationPresets READ translationPresets CONSTANT)
|
||||||
Q_PROPERTY(bool fetchingModels READ isFetchingModels NOTIFY fetchingModelsChanged)
|
Q_PROPERTY(bool fetchingModels READ isFetchingModels NOTIFY fetchingModelsChanged)
|
||||||
|
Q_PROPERTY(QVariantList glossaryEntries READ glossaryEntries NOTIFY glossaryChanged)
|
||||||
|
Q_PROPERTY(QString glossaryDirectory READ glossaryDirectory NOTIFY glossaryChanged)
|
||||||
|
|
||||||
public:
|
public:
|
||||||
explicit TranslationService(QObject *parent = nullptr);
|
explicit TranslationService(QObject *parent = nullptr);
|
||||||
@@ -43,6 +46,8 @@ public:
|
|||||||
QStringList availableModels() const;
|
QStringList availableModels() const;
|
||||||
QVariantList translationPresets() const;
|
QVariantList translationPresets() const;
|
||||||
bool isFetchingModels() const;
|
bool isFetchingModels() const;
|
||||||
|
QVariantList glossaryEntries() const;
|
||||||
|
QString glossaryDirectory() const;
|
||||||
|
|
||||||
// \a track is an entry of MpvItem::subtitleTracks.
|
// \a track is an entry of MpvItem::subtitleTracks.
|
||||||
Q_INVOKABLE void translate(const QUrl &videoUrl, const QVariantMap &track, const QString &mediaTitle);
|
Q_INVOKABLE void translate(const QUrl &videoUrl, const QVariantMap &track, const QString &mediaTitle);
|
||||||
@@ -50,6 +55,9 @@ public:
|
|||||||
// Asks the API configured in the settings for its models.
|
// Asks the API configured in the settings for its models.
|
||||||
Q_INVOKABLE void fetchModels();
|
Q_INVOKABLE void fetchModels();
|
||||||
Q_INVOKABLE void clearError();
|
Q_INVOKABLE void clearError();
|
||||||
|
Q_INVOKABLE void loadGlossary(const QUrl &videoUrl);
|
||||||
|
Q_INVOKABLE void saveGlossary(const QVariantList &entries);
|
||||||
|
Q_INVOKABLE void clearGlossary();
|
||||||
|
|
||||||
Q_SIGNALS:
|
Q_SIGNALS:
|
||||||
void busyChanged();
|
void busyChanged();
|
||||||
@@ -58,6 +66,7 @@ Q_SIGNALS:
|
|||||||
void errorTextChanged();
|
void errorTextChanged();
|
||||||
void availableModelsChanged();
|
void availableModelsChanged();
|
||||||
void fetchingModelsChanged();
|
void fetchingModelsChanged();
|
||||||
|
void glossaryChanged();
|
||||||
void translated(const QUrl &subtitleUrl, bool fromCache);
|
void translated(const QUrl &subtitleUrl, bool fromCache);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
@@ -91,4 +100,5 @@ private:
|
|||||||
QString m_errorText;
|
QString m_errorText;
|
||||||
QStringList m_availableModels;
|
QStringList m_availableModels;
|
||||||
bool m_fetchingModels = false;
|
bool m_fetchingModels = false;
|
||||||
|
GlossaryStore::Glossary m_glossary;
|
||||||
};
|
};
|
||||||
Reference in new issue
Block a user