refactor: убрать ограничение maxChars из хранилища соответствий
- Удалены поле maxChars, параметр конструктора и метод evictWhileOverLimit. - Хранилище теперь ограничено только TTL (ttl-minutes), без вытеснения по объёму. - Конструкторы переведены на (int ttlMinutes) и (int ttlMinutes, SharedIndex, PayloadCipher). - Обновлены тесты и PipelineWarmup на новые сигнатуры.
This commit is contained in:
@@ -15,7 +15,7 @@ import ru.pdguard.mask.Masker;
|
||||
class BankTypesTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private void assertHidden(String text, String secret) {
|
||||
String masked = pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -80,7 +80,7 @@ class BenchmarkTest {
|
||||
}
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(10_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
/**
|
||||
* Набор, на котором правила отлаживались. Пороги здесь высокие: любое падение означает, что
|
||||
@@ -131,7 +131,7 @@ class BenchmarkTest {
|
||||
? new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(ENGINE, Optional.of(MODEL_PATH), 16, 4))
|
||||
: pipeline;
|
||||
Result result = measure(stage, "/benchmark-holdout3.txt", "второй контрольный набор");
|
||||
@@ -158,7 +158,7 @@ class BenchmarkTest {
|
||||
? new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(ENGINE, Optional.of(MODEL_PATH), 16, 4))
|
||||
: pipeline;
|
||||
Result result = measure(stage, "/benchmark-holdout2.txt", "второй отложенный набор");
|
||||
@@ -206,7 +206,7 @@ class BenchmarkTest {
|
||||
? new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(ENGINE, Optional.of(MODEL_PATH), 16, 4))
|
||||
: pipeline;
|
||||
Result result = measure(stage, "/benchmark-generated.txt", "сгенерированный набор");
|
||||
@@ -232,7 +232,7 @@ class BenchmarkTest {
|
||||
new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(ENGINE, Optional.of(MODEL_PATH), 16, 4));
|
||||
Result result =
|
||||
measure(withCascade, "/benchmark-holdout.txt", "отложенный набор, вторая ступень включена");
|
||||
|
||||
@@ -16,7 +16,7 @@ import ru.pdguard.mask.Masker;
|
||||
class ContextDetectionTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -48,7 +48,7 @@ class Dataset200Test {
|
||||
}
|
||||
|
||||
private void runCase(BenchmarkFixtures.Sample sample, int index) {
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(1_000_000L, 30));
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(30));
|
||||
String payloadId = "dataset200-" + index;
|
||||
|
||||
String masked = pipeline.process(sample.text(), payloadId, SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -17,7 +17,7 @@ import ru.pdguard.mask.Masker;
|
||||
class DateAndAddressTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -16,7 +16,7 @@ import ru.pdguard.mask.Masker;
|
||||
class FioTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -110,7 +110,7 @@ class HugeDatasetTest {
|
||||
|
||||
private void runCase(int targetChars, int seed) {
|
||||
Pipeline pipeline =
|
||||
new Pipeline(REGISTRY, MASKER, new PayloadStore(targetChars * 2L + 4096, 30));
|
||||
new Pipeline(REGISTRY, MASKER, new PayloadStore(30));
|
||||
BenchmarkFixtures.Sample sample = buildText(targetChars, seed);
|
||||
String payloadId = "dataset-" + seed;
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ import ru.pdguard.mask.Masker;
|
||||
class IdentityDocumentTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private void assertHidden(String text, String secret) {
|
||||
String masked = pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -74,7 +74,7 @@ class LargeTextTest {
|
||||
BenchmarkFixtures.Sample large = buildLargeText(TARGET_CHARS, 1);
|
||||
Pipeline pipeline =
|
||||
new Pipeline(
|
||||
new RuleRegistry(), new Masker(), new PayloadStore(large.text().length() * 2L, 30));
|
||||
new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
long maskStarted = System.nanoTime();
|
||||
String masked = pipeline.process(large.text(), "large-mixed-1", SystemPolicy.DEFAULT);
|
||||
@@ -102,7 +102,7 @@ class LargeTextTest {
|
||||
@Test
|
||||
void recallHoldsAtScale() {
|
||||
BenchmarkFixtures.Sample large = buildLargeText(TARGET_CHARS, 2);
|
||||
Pipeline pipeline = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1L, 30));
|
||||
Pipeline pipeline = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
List<Span> found = pipeline.findPersonalData(large.text(), SystemPolicy.DEFAULT);
|
||||
int hit = 0;
|
||||
@@ -138,7 +138,7 @@ class LargeTextTest {
|
||||
new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(large.text().length() * 2L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(ENGINE, Optional.of(MODEL_PATH), 16, 4));
|
||||
|
||||
long started = System.nanoTime();
|
||||
|
||||
@@ -31,7 +31,7 @@ class LeakDiagTest {
|
||||
|
||||
@Test
|
||||
void checkLeaks() {
|
||||
Pipeline p = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
Pipeline p = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
List<String> leaks = readDataset();
|
||||
|
||||
int fixed = 0;
|
||||
|
||||
@@ -25,7 +25,7 @@ class LegalNerTest {
|
||||
"off", Optional.empty(),
|
||||
"ru-legal-ner", Optional.of("models/ru-legal-ner")),
|
||||
16, 4);
|
||||
Pipeline p = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30), cascade);
|
||||
Pipeline p = new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30), cascade);
|
||||
return p.findPersonalData(text, SystemPolicy.DEFAULT);
|
||||
}
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ import ru.pdguard.mask.Masker;
|
||||
class MaskModeTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private SystemPolicy policy(MaskMode mode) {
|
||||
return new SystemPolicy(
|
||||
|
||||
@@ -24,7 +24,7 @@ class NameCascadeTest {
|
||||
|
||||
private String mask(NameCascade cascade, String payloadId) {
|
||||
Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30), cascade);
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30), cascade);
|
||||
return pipeline.process(TEXT, payloadId, SystemPolicy.DEFAULT);
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
class NormalisedDigitsTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private void assertHidden(String text, String secret) {
|
||||
String masked = pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -18,7 +18,7 @@ import ru.pdguard.mask.Masker;
|
||||
class OrganisationNamesTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -35,7 +35,7 @@ class PayloadCipherTest {
|
||||
void storeStoresEncryptedButReturnsPlaintext() {
|
||||
PayloadCipher cipher = new PayloadCipher(KEY);
|
||||
PayloadStore store =
|
||||
new PayloadStore(1_000_000L, 30, ru.pdguard.core.SharedIndex.disabled(), cipher);
|
||||
new PayloadStore(30, ru.pdguard.core.SharedIndex.disabled(), cipher);
|
||||
|
||||
String original = "Клиент Иванов Иван Иванович, паспорт 4509 123456";
|
||||
String masked = "Клиент И. И. И., паспорт 45** ****56";
|
||||
|
||||
@@ -3,19 +3,18 @@ package ru.pdguard;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import ru.pdguard.core.PayloadStore;
|
||||
|
||||
/** Ограничения хранилища соответствий: объём, срок жизни и разделение по системам. */
|
||||
/** Ограничения хранилища соответствий: срок жизни и разделение по системам. */
|
||||
class PayloadStoreTest {
|
||||
|
||||
private static final String SYSTEM = "crm";
|
||||
|
||||
@Test
|
||||
void returnsWhatWasStored() {
|
||||
PayloadStore store = new PayloadStore(1_000_000L, 30);
|
||||
PayloadStore store = new PayloadStore(30);
|
||||
store.put(SYSTEM, "id", "исходный текст", "маска");
|
||||
|
||||
PayloadStore.Entry entry = store.byId(SYSTEM, "id");
|
||||
@@ -27,28 +26,16 @@ class PayloadStoreTest {
|
||||
|
||||
@Test
|
||||
void forgetsEntriesAfterTheirLifetime() {
|
||||
PayloadStore store = new PayloadStore(1_000_000L, 0);
|
||||
PayloadStore store = new PayloadStore(0);
|
||||
store.put(SYSTEM, "id", "исходный текст", "маска");
|
||||
|
||||
assertNull(store.byId(SYSTEM, "id"), "запись с истёкшим сроком жизни не должна отдаваться");
|
||||
assertNull(store.originalForMask(SYSTEM, "маска"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void evictsOldestWhenOverSizeLimit() {
|
||||
PayloadStore store = new PayloadStore(100L, 30);
|
||||
for (int i = 0; i < 50; i++) {
|
||||
store.put(SYSTEM, "id" + i, "текст номер " + i, "маска номер " + i);
|
||||
}
|
||||
|
||||
assertTrue(store.charsHeld() <= 100, "объём хранилища вышел за предел: " + store.charsHeld());
|
||||
assertNull(store.byId(SYSTEM, "id0"), "самая старая запись должна быть вытеснена");
|
||||
assertNotNull(store.byId(SYSTEM, "id49"), "последняя запись должна остаться");
|
||||
}
|
||||
|
||||
@Test
|
||||
void unknownKeysReturnNothing() {
|
||||
PayloadStore store = new PayloadStore(1_000_000L, 30);
|
||||
PayloadStore store = new PayloadStore(30);
|
||||
assertNull(store.byId(SYSTEM, "нет такого"));
|
||||
assertNull(store.originalForMask(SYSTEM, "нет такой маски"));
|
||||
}
|
||||
@@ -60,7 +47,7 @@ class PayloadStoreTest {
|
||||
*/
|
||||
@Test
|
||||
void oneSystemCannotReadAnotherSystemData() {
|
||||
PayloadStore store = new PayloadStore(1_000_000L, 30);
|
||||
PayloadStore store = new PayloadStore(30);
|
||||
store.put("crm", "общий-id", "Иванов Иван Иванович", "И. И. И.");
|
||||
|
||||
assertNull(
|
||||
|
||||
@@ -25,7 +25,7 @@ import ru.pdguard.mask.Masker;
|
||||
class PdnTypeEfficiencyTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(10_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
/** Накопитель посимвольных совпадений по одному типу. */
|
||||
private static final class Score {
|
||||
|
||||
@@ -60,7 +60,7 @@ class PerformanceBenchmarkTest {
|
||||
private static final int MEASURE = 50_000;
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(10_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private void warmup() {
|
||||
for (int i = 0; i < WARMUP; i++) {
|
||||
@@ -184,7 +184,7 @@ class PerformanceBenchmarkTest {
|
||||
new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(
|
||||
"rubert", Optional.of(model.toString()), "off", Optional.empty(), 16, 4, meters));
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ class PipelineTest {
|
||||
private static final String VALID_SNILS = "112-233-445 95";
|
||||
|
||||
private Pipeline pipeline() {
|
||||
return new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
return new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
}
|
||||
|
||||
private String mask(Pipeline pipeline, String text) {
|
||||
|
||||
@@ -74,7 +74,7 @@ class PlacementVariantsTest {
|
||||
|
||||
/** Демаскирование обязано восстановить исходный текст всегда, независимо от утечек. */
|
||||
private void runRoundTrip(BenchmarkFixtures.Sample sample, int index) {
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(1_000_000L, 30), CASCADE);
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(30), CASCADE);
|
||||
String payloadId = "placement-" + index;
|
||||
|
||||
String masked = pipeline.process(sample.text(), payloadId, SystemPolicy.DEFAULT);
|
||||
@@ -90,7 +90,7 @@ class PlacementVariantsTest {
|
||||
*/
|
||||
@Test
|
||||
void leakSummary() {
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(10_000_000L, 30), CASCADE);
|
||||
Pipeline pipeline = new Pipeline(REGISTRY, MASKER, new PayloadStore(30), CASCADE);
|
||||
int leaked = 0;
|
||||
int checked = 0;
|
||||
java.util.Map<String, Integer> byType = new java.util.LinkedHashMap<>();
|
||||
|
||||
@@ -20,7 +20,7 @@ import ru.pdguard.mask.Masker;
|
||||
class SettlementTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -21,7 +21,7 @@ import ru.pdguard.mask.Masker;
|
||||
class StreetDenylistTest {
|
||||
|
||||
private final Pipeline pipeline =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30));
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30));
|
||||
|
||||
private String mask(String text) {
|
||||
return pipeline.process(text, UUID.randomUUID().toString(), SystemPolicy.DEFAULT);
|
||||
|
||||
@@ -29,7 +29,7 @@ class TwoModelBenchmarkTest {
|
||||
new Pipeline(
|
||||
new RuleRegistry(),
|
||||
new Masker(),
|
||||
new PayloadStore(10_000_000L, 30),
|
||||
new PayloadStore(30),
|
||||
new NameCascade(
|
||||
new NameCascade.EngineConfig(
|
||||
"wikineural", Optional.of("models/wikineural-ner"),
|
||||
|
||||
@@ -27,7 +27,7 @@ class TwoModelCascadeTest {
|
||||
16,
|
||||
4);
|
||||
Pipeline p =
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(1_000_000L, 30), cascade);
|
||||
new Pipeline(new RuleRegistry(), new Masker(), new PayloadStore(30), cascade);
|
||||
return p.findPersonalData(text, SystemPolicy.DEFAULT);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user