feat(translate): add per-series glossary with auto extraction

src/translate/translationservice.cpp

@@ -127,6 +127,48 @@ bool TranslationService::isFetchingModels() const
     return m_fetchingModels;
 }
 
+QVariantList TranslationService::glossaryEntries() const
+{
+    QVariantList result;
+    for (const auto &entry : m_glossary.entries) result.append(QVariantMap{{u"source"_s, entry.source}, {u"target"_s, entry.target}});
+    return result;
+}
+
+QString TranslationService::glossaryDirectory() const
+{
+    return m_glossary.directory;
+}
+
+void TranslationService::loadGlossary(const QUrl &videoUrl)
+{
+    const QString directory = videoUrl.isLocalFile() ? QFileInfo(videoUrl.toLocalFile()).absolutePath() : QString();
+    m_glossary = directory.isEmpty() ? GlossaryStore::Glossary{} : GlossaryStore::load(directory);
+    Q_EMIT glossaryChanged();
+}
+
+void TranslationService::saveGlossary(const QVariantList &entries)
+{
+    QList<GlossaryStore::Entry> result;
+    for (const QVariant &value : entries) {
+        const QVariantMap map = value.toMap();
+        result.append({map.value(u"source"_s).toString().trimmed(), map.value(u"target"_s).toString().trimmed()});
+    }
+    m_glossary.entries = result;
+    if (!m_glossary.directory.isEmpty()) {
+        GlossaryStore::save(m_glossary);
+    }
+    Q_EMIT glossaryChanged();
+}
+
+void TranslationService::clearGlossary()
+{
+    m_glossary.entries.clear();
+    if (!m_glossary.directory.isEmpty()) {
+        GlossaryStore::save(m_glossary);
+    }
+    Q_EMIT glossaryChanged();
+}
+
 void TranslationService::translate(const QUrl &videoUrl, const QVariantMap &track, const QString &mediaTitle)
 {
     cancel();
@@ -172,6 +214,7 @@ QCoro::Task<std::pair<LlmClient::Config, QString>> TranslationService::loadConfi
         settings->targetLanguage().trimmed(),
         settings->translationPreset().trimmed(),
         settings->customInstructions(),
+        {},
     };
 
     if (!config.baseUrl.isValid() || config.baseUrl.scheme().isEmpty() || config.baseUrl.host().isEmpty()) {
@@ -220,7 +263,7 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
     setBusy(true);
     setStatusText(i18n("Reading subtitles…"));
 
-    const auto [config, configError] = co_await loadConfig();
+    auto [config, configError] = co_await loadConfig();
     if (job->cancelled) {
         co_return;
     }
@@ -247,15 +290,25 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
         co_return;
     }
 
-    // Same subtitles, model, language, and prompt settings give the same file, so it is translated once.
-    QCryptographicHash hash(QCryptographicHash::Sha1);
-    hash.addData(CacheVersion);
-    hash.addData(config.model.toUtf8());
-    hash.addData(config.targetLanguage.toUtf8());
-    hash.addData(config.translationPreset.toUtf8());
-    hash.addData(config.customInstructions.toUtf8());
-    hash.addData(Subtitles::toSrt(cues).toUtf8());
-    const QString outputPath = cachePath(videoUrl, QString::fromLatin1(hash.result().toHex().left(12)), config.targetLanguage);
+    // Episodes in one folder share a glossary. Streams have no folder, so they get none.
+    const QString seriesDirectory = videoUrl.isLocalFile() ? QFileInfo(videoUrl.toLocalFile()).absolutePath() : QString();
+    const bool useGlossary = !seriesDirectory.isEmpty();
+    GlossaryStore::Glossary glossary = useGlossary ? GlossaryStore::load(seriesDirectory) : GlossaryStore::Glossary{};
+    const QString sourceSrt = Subtitles::toSrt(cues);
+
+    // Same subtitles, model, language, prompt settings and glossary give the same file, so it is translated once.
+    const auto outputPathFor = [&](const GlossaryStore::Glossary &g) {
+        QCryptographicHash hash(QCryptographicHash::Sha1);
+        hash.addData(CacheVersion);
+        hash.addData(config.model.toUtf8());
+        hash.addData(config.targetLanguage.toUtf8());
+        hash.addData(config.translationPreset.toUtf8());
+        hash.addData(config.customInstructions.toUtf8());
+        hash.addData(GlossaryStore::serialize(g).toUtf8());
+        hash.addData(sourceSrt.toUtf8());
+        return cachePath(videoUrl, QString::fromLatin1(hash.result().toHex().left(12)), config.targetLanguage);
+    };
+    QString outputPath = outputPathFor(glossary);
 
     if (QFileInfo::exists(outputPath)) {
         m_job.reset();
@@ -265,6 +318,55 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
         co_return;
     }
 
+    if (useGlossary && QtPotatoSettings::self()->autoBuildGlossary()) {
+        setStatusText(i18n("Building glossary…"));
+        QStringList extractionLines;
+        QHash<QString, bool> seen;
+        for (const SubtitleCue &cue : cues) {
+            if (!seen.contains(cue.text)) { seen.insert(cue.text, true); extractionLines.append(cue.text); }
+        }
+        constexpr qsizetype MaxGlossaryChars = 30000;
+        // Long subtitles are sampled evenly so names from the whole episode are seen.
+        qsizetype allChars = 0;
+        for (const QString &line : std::as_const(extractionLines)) {
+            allChars += line.size() + 1;
+        }
+        const qsizetype step = std::max<qsizetype>(1, (allChars + MaxGlossaryChars - 1) / MaxGlossaryChars);
+        QStringList sampled;
+        qsizetype total = 0;
+        for (qsizetype i = 0; i < extractionLines.size(); i += step) {
+            if (total + extractionLines.at(i).size() + 1 > MaxGlossaryChars) {
+                break;
+            }
+            sampled.append(extractionLines.at(i));
+            total += extractionLines.at(i).size() + 1;
+        }
+        job->client = std::make_unique<LlmClient>(m_network, config);
+        const LlmClient::BatchResult extracted = co_await job->client->extractGlossary(sampled, job->mediaTitle);
+        if (job->cancelled) {
+            co_return;
+        }
+        if (extracted.error.isEmpty()) {
+            QList<GlossaryStore::Entry> additions;
+            for (const QString &encoded : extracted.translations) {
+                const qsizetype tab = encoded.indexOf(u'\t');
+                if (tab > 0) additions.append({encoded.left(tab), encoded.mid(tab + 1)});
+            }
+            glossary = GlossaryStore::merge(glossary, additions);
+            GlossaryStore::save(glossary);
+        } else {
+            setStatusText(i18n("Glossary unavailable; translating without it."));
+        }
+        outputPath = outputPathFor(glossary);
+        if (QFileInfo::exists(outputPath)) {
+            m_job.reset();
+            setBusy(false);
+            setStatusText({});
+            Q_EMIT translated(QUrl::fromLocalFile(outputPath), true);
+            co_return;
+        }
+    }
+
     // Repeated lines (songs, catchphrases) are sent once.
     QStringList uniqueLines;
     QHash<QString, qsizetype> uniqueIndex;
@@ -278,6 +380,12 @@ QCoro::Task<> TranslationService::run(QUrl videoUrl, QVariantMap track)
         cueToUnique.append(*it);
     }
 
+    QStringList glossaryLines;
+    for (const auto &entry : std::as_const(glossary.entries)) {
+        glossaryLines.append(entry.source + u" → "_s + entry.target);
+    }
+    config.glossaryText = glossaryLines.join(u'\n');
+
     job->client = std::make_unique<LlmClient>(m_network, config);
     auto batches = std::make_shared<Batches>();
     batches->lines = uniqueLines;