fix(worker): filter the transliterated 'fuer' stopword, not the untransliterated 'fur'
NormalizeTitleWords transliterates umlauts before tokenizing ('für' ->
'fuer'), but TitleStopWords listed 'fur' -- a token the pipeline can
never produce -- so 'für' was never filtered as filler and could push
unrelated titles past the near-duplicate threshold.
This commit is contained in:
+1
-1
@@ -329,7 +329,7 @@ public sealed class ExternalMcpService
|
||||
// plus the recurring "mcp"/"task(s)"/"cleanup" nouns called out in the task write-up).
|
||||
private static readonly HashSet<string> TitleStopWords = new(StringComparer.Ordinal)
|
||||
{
|
||||
"der", "die", "das", "und", "von", "auf", "mit", "fur", "ein", "eine",
|
||||
"der", "die", "das", "und", "von", "auf", "mit", "fuer", "ein", "eine",
|
||||
"ist", "sind", "oder", "nicht", "mcp", "task", "tasks", "cleanup",
|
||||
};
|
||||
|
||||
|
||||
@@ -1829,6 +1829,24 @@ public sealed class ExternalMcpServiceTests : IDisposable
|
||||
Assert.Empty(result.PossibleDuplicates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddTask_TitlesSharingOnlyOneWordPlusTheGermanStopwordFuer_AreNotReportedAsDuplicates()
|
||||
{
|
||||
// Regression for the "fur" (never-transliterated) stopword typo: NormalizeTitleWords
|
||||
// transliterates "für" -> "fuer" before the stopword check, so a stopword list containing
|
||||
// "fur" can never filter it. Before the fix these two titles shared "report" and "fuer"
|
||||
// (2 words, 50% of the shorter title) and cleared the duplicate threshold undeservedly;
|
||||
// with "fuer" correctly filtered as filler only "report" remains shared (1 word).
|
||||
var listId = await SeedListAsync();
|
||||
await SeedTaskAsync(listId, "Neuer Report für Bestand", TaskStatus.Idle);
|
||||
var sut = NewService();
|
||||
|
||||
var result = await sut.AddTask(
|
||||
listId, "Anderer Report für Kunden", cancellationToken: CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.PossibleDuplicates);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task AddTask_MoreThanThreeMatches_CapsPossibleDuplicatesAtThree()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user