fix(worker): filter the transliterated 'fuer' stopword, not the untransliterated 'fur'
NormalizeTitleWords transliterates umlauts before tokenizing ('für' ->
'fuer'), but TitleStopWords listed 'fur' -- a token the pipeline can
never produce -- so 'für' was never filtered as filler and could push
unrelated titles past the near-duplicate threshold.
This commit is contained in:
+1
-1
@@ -329,7 +329,7 @@ public sealed class ExternalMcpService
|
|||||||
// plus the recurring "mcp"/"task(s)"/"cleanup" nouns called out in the task write-up).
|
// plus the recurring "mcp"/"task(s)"/"cleanup" nouns called out in the task write-up).
|
||||||
private static readonly HashSet<string> TitleStopWords = new(StringComparer.Ordinal)
|
private static readonly HashSet<string> TitleStopWords = new(StringComparer.Ordinal)
|
||||||
{
|
{
|
||||||
"der", "die", "das", "und", "von", "auf", "mit", "fur", "ein", "eine",
|
"der", "die", "das", "und", "von", "auf", "mit", "fuer", "ein", "eine",
|
||||||
"ist", "sind", "oder", "nicht", "mcp", "task", "tasks", "cleanup",
|
"ist", "sind", "oder", "nicht", "mcp", "task", "tasks", "cleanup",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|||||||
@@ -1829,6 +1829,24 @@ public sealed class ExternalMcpServiceTests : IDisposable
|
|||||||
Assert.Empty(result.PossibleDuplicates);
|
Assert.Empty(result.PossibleDuplicates);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task AddTask_TitlesSharingOnlyOneWordPlusTheGermanStopwordFuer_AreNotReportedAsDuplicates()
|
||||||
|
{
|
||||||
|
// Regression for the "fur" (never-transliterated) stopword typo: NormalizeTitleWords
|
||||||
|
// transliterates "für" -> "fuer" before the stopword check, so a stopword list containing
|
||||||
|
// "fur" can never filter it. Before the fix these two titles shared "report" and "fuer"
|
||||||
|
// (2 words, 50% of the shorter title) and cleared the duplicate threshold undeservedly;
|
||||||
|
// with "fuer" correctly filtered as filler only "report" remains shared (1 word).
|
||||||
|
var listId = await SeedListAsync();
|
||||||
|
await SeedTaskAsync(listId, "Neuer Report für Bestand", TaskStatus.Idle);
|
||||||
|
var sut = NewService();
|
||||||
|
|
||||||
|
var result = await sut.AddTask(
|
||||||
|
listId, "Anderer Report für Kunden", cancellationToken: CancellationToken.None);
|
||||||
|
|
||||||
|
Assert.Empty(result.PossibleDuplicates);
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task AddTask_MoreThanThreeMatches_CapsPossibleDuplicatesAtThree()
|
public async Task AddTask_MoreThanThreeMatches_CapsPossibleDuplicatesAtThree()
|
||||||
{
|
{
|
||||||
|
|||||||
Reference in New Issue
Block a user