Added message uningest to reverse Markov data when recently ingested messages are deleted.

tests/test_cache.py

@@ -0,0 +1,118 @@
+# Copyright 2026 Logan Fick
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Unit tests for the IngestCache TTL cache."""
+
+import asyncio
+import time
+
+from crabstero.cache import CachedMessage, IngestCache
+
+
+class TestIngestCache:
+    """IngestCache put, pop, and expiry behavior."""
+
+    def test_put_and_pop(self) -> None:
+        """A cached message can be retrieved by message ID."""
+        cache = IngestCache()
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello world",
+            embed_texts=["embed title"],
+            image_urls=["https://example.com/cat.png"],
+        )
+        cache.put(12345, entry)
+        assert cache.pop(12345) is entry
+
+    def test_pop_removes_entry(self) -> None:
+        """Popping an entry removes it from the cache."""
+        cache = IngestCache()
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello",
+            embed_texts=[],
+            image_urls=[],
+        )
+        cache.put(12345, entry)
+        cache.pop(12345)
+        assert cache.pop(12345) is None
+
+    def test_pop_returns_none_for_missing(self) -> None:
+        """Popping a nonexistent key returns None."""
+        cache = IngestCache()
+        assert cache.pop(99999) is None
+
+    def test_expired_entry_not_returned(self) -> None:
+        """An expired entry is not returned by pop."""
+        cache = IngestCache(ttl_seconds=0.01)
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello",
+            embed_texts=[],
+            image_urls=[],
+        )
+        cache.put(12345, entry)
+        time.sleep(0.02)
+        assert cache.pop(12345) is None
+
+    def test_cleanup_removes_expired(self) -> None:
+        """Cleanup evicts all expired entries."""
+        cache = IngestCache(ttl_seconds=0.01)
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello",
+            embed_texts=[],
+            image_urls=[],
+        )
+        cache.put(1, entry)
+        cache.put(2, entry)
+        time.sleep(0.02)
+        cache._cleanup()
+        assert cache.pop(1) is None
+        assert cache.pop(2) is None
+
+    def test_cleanup_keeps_unexpired(self) -> None:
+        """Cleanup does not evict entries that are still valid."""
+        cache = IngestCache(ttl_seconds=300)
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello",
+            embed_texts=[],
+            image_urls=[],
+        )
+        cache.put(1, entry)
+        cache._cleanup()
+        assert cache.pop(1) is entry
+
+    async def test_start_and_stop(self) -> None:
+        """The background cleanup task can be started and stopped."""
+        cache = IngestCache(ttl_seconds=0, cleanup_interval_seconds=0.01)
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello",
+            embed_texts=[],
+            image_urls=[],
+        )
+        cache.put(1, entry)
+        cache.start()
+        await asyncio.sleep(0.05)
+        cache.stop()
+        # Entry should have been cleaned up by the background task.
+        assert cache.pop(1) is None

tests/test_database.py

@@ -127,7 +127,7 @@ class TestImages:
 
     async def test_add_and_retrieve(self, db: Database) -> None:
         """Inserted image URL can be retrieved by channel."""
-        await db.add_image(1, 100, "https://example.com/cat.png")
+        await db.add_images([(1, 100, "https://example.com/cat.png")])
         result = await db.get_random_image(1)
         assert result == "https://example.com/cat.png"
 
@@ -181,6 +181,103 @@ class TestChannelIngestion:
         assert await db.is_channel_ingested(42) is True
 
 
+class TestRemoveMarkovData:
+    """Markov data removal via remove_markov_data."""
+
+    async def test_removes_one_start_word(self, db: Database) -> None:
+        """Removes exactly one matching start word row."""
+        await db.add_markov_data([(1, 100, "Hello"), (1, 100, "Hello")], [])
+        await db.remove_markov_data([(1, 100, "Hello")], [])
+        # One copy should remain.
+        assert await db.get_random_start_word(1) == "Hello"
+
+    async def test_removes_one_transition(self, db: Database) -> None:
+        """Removes exactly one matching transition row."""
+        await db.add_markov_data(
+            [], [(1, 100, "Hello", "world."), (1, 100, "Hello", "world.")]
+        )
+        await db.remove_markov_data([], [(1, 100, "Hello", "world.")])
+        assert await db.get_random_next_word(1, "Hello") == "world."
+
+    async def test_removes_last_start_word(self, db: Database) -> None:
+        """Removing the only start word leaves the table empty for that channel."""
+        await db.add_markov_data([(1, 100, "Hello")], [])
+        await db.remove_markov_data([(1, 100, "Hello")], [])
+        assert await db.get_random_start_word(1) is None
+
+    async def test_removes_last_transition(self, db: Database) -> None:
+        """Removing the only transition leaves no next word."""
+        await db.add_markov_data([], [(1, 100, "Hello", "world.")])
+        await db.remove_markov_data([], [(1, 100, "Hello", "world.")])
+        assert await db.get_random_next_word(1, "Hello") is None
+
+    async def test_no_match_is_noop(self, db: Database) -> None:
+        """Removing a non-existent row does not raise."""
+        await db.remove_markov_data([(1, 100, "nope")], [(1, 100, "nope", "nah")])
+
+    async def test_start_word_removal_scoped_to_channel(self, db: Database) -> None:
+        """Removing a start word in one channel leaves another channel intact."""
+        await db.add_markov_data([(1, 100, "Hello"), (2, 200, "Hello")], [])
+        await db.remove_markov_data([(1, 100, "Hello")], [])
+        assert await db.get_random_start_word(1) is None
+        assert await db.get_random_start_word(2) == "Hello"
+
+    async def test_transition_removal_scoped_to_channel(self, db: Database) -> None:
+        """Removing a transition in one channel leaves another channel intact."""
+        await db.add_markov_data(
+            [], [(1, 100, "Hello", "world."), (2, 200, "Hello", "world.")]
+        )
+        await db.remove_markov_data([], [(1, 100, "Hello", "world.")])
+        assert await db.get_random_next_word(1, "Hello") is None
+        assert await db.get_random_next_word(2, "Hello") == "world."
+
+    async def test_empty_lists_is_noop(self, db: Database) -> None:
+        """Empty lists do not error."""
+        await db.remove_markov_data([], [])
+
+
+class TestRemoveImages:
+    """Image removal via remove_images."""
+
+    async def test_removes_one_image(self, db: Database) -> None:
+        """Removes exactly one matching image row."""
+        await db.add_images(
+            [
+                (1, 100, "https://example.com/a.png"),
+                (1, 100, "https://example.com/a.png"),
+            ]
+        )
+        await db.remove_images([(1, 100, "https://example.com/a.png")])
+        # One copy should remain.
+        assert await db.get_random_image(1) == "https://example.com/a.png"
+
+    async def test_removes_last_image(self, db: Database) -> None:
+        """Removing the only image leaves none for that channel."""
+        await db.add_images([(1, 100, "https://example.com/a.png")])
+        await db.remove_images([(1, 100, "https://example.com/a.png")])
+        assert await db.get_random_image(1) is None
+
+    async def test_image_removal_scoped_to_channel(self, db: Database) -> None:
+        """Removing an image in one channel leaves another channel intact."""
+        await db.add_images(
+            [
+                (1, 100, "https://example.com/a.png"),
+                (2, 200, "https://example.com/a.png"),
+            ]
+        )
+        await db.remove_images([(1, 100, "https://example.com/a.png")])
+        assert await db.get_random_image(1) is None
+        assert await db.get_random_image(2) == "https://example.com/a.png"
+
+    async def test_no_match_is_noop(self, db: Database) -> None:
+        """Removing a non-existent image does not raise."""
+        await db.remove_images([(1, 100, "https://example.com/nope.png")])
+
+    async def test_empty_list_is_noop(self, db: Database) -> None:
+        """Empty list does not error."""
+        await db.remove_images([])
+
+
 class TestWriteDurability:
     """Writes persist across close and reopen."""
 

tests/test_markov.py

@@ -25,9 +25,11 @@ import pytest
 from crabstero.markov import (
     DEFAULT_SENTENCE_END,
     _ingest_sentence,
+    _uningest_sentence,
     generate,
     ingest,
     is_complete_sentence,
+    uningest,
 )
 
 if TYPE_CHECKING:
@@ -101,7 +103,7 @@ class TestIngestSentence:
         assert await db.get_random_next_word(1, "B") == "C."
 
 
-class TestIngest:
+class TestIngestParagraph:
     """Paragraph ingestion splits into sentences."""
 
     async def test_single_sentence(self, db: Database) -> None:
@@ -287,3 +289,50 @@ class TestGenerate:
             " Chat a bit more so I can learn how this channel talks!"
         )
         assert result == expected
+
+
+class TestUningestSentence:
+    """Single sentence uningest from the Markov chain."""
+
+    async def test_removes_start_word_and_transition(self, db: Database) -> None:
+        """Uningest removes the start word and transition added by ingest."""
+        await _ingest_sentence(db, 1, 100, "Hello world.")
+        await _uningest_sentence(db, 1, 100, "Hello world.")
+        assert await db.get_random_start_word(1) is None
+        assert await db.get_random_next_word(1, "Hello") is None
+
+    async def test_preserves_other_data(self, db: Database) -> None:
+        """Uningest only removes data for the specified sentence."""
+        await _ingest_sentence(db, 1, 100, "Hello world.")
+        await _ingest_sentence(db, 1, 100, "Goodbye world.")
+        await _uningest_sentence(db, 1, 100, "Hello world.")
+        assert await db.get_random_start_word(1) == "Goodbye"
+
+    async def test_handles_missing_punctuation(self, db: Database) -> None:
+        """Uningest appends the default sentence end, matching ingest behavior."""
+        await _ingest_sentence(db, 1, 100, "Hello world")
+        await _uningest_sentence(db, 1, 100, "Hello world")
+        assert await db.get_random_start_word(1) is None
+
+    async def test_single_word_is_noop(self, db: Database) -> None:
+        """Uningesting a single-word sentence does not error."""
+        await _ingest_sentence(db, 1, 100, "Hello.")
+        await _uningest_sentence(db, 1, 100, "Hello.")
+
+
+class TestUningestParagraph:
+    """Paragraph-level uningest."""
+
+    async def test_uningest_multiple_sentences(self, db: Database) -> None:
+        """Uningest reverses a multi-sentence paragraph."""
+        await ingest(db, 1, 100, "Hello world. Goodbye world!")
+        await uningest(db, 1, 100, "Hello world. Goodbye world!")
+        assert await db.get_random_start_word(1) is None
+
+    async def test_uningest_preserves_duplicate_data(self, db: Database) -> None:
+        """Uningesting one copy leaves the other intact."""
+        await ingest(db, 1, 100, "Hello world.")
+        await ingest(db, 1, 100, "Hello world.")
+        await uningest(db, 1, 100, "Hello world.")
+        assert await db.get_random_start_word(1) == "Hello"
+        assert await db.get_random_next_word(1, "Hello") == "world."

tests/test_metrics.py

@@ -72,6 +72,9 @@ class TestMetricObjects:
                 "crabstero_channel_ingestion_messages", id="channel-ingestion-messages"
             ),
             pytest.param("crabstero_discord_events_total", id="discord-events"),
+            pytest.param(
+                "crabstero_messages_uningested_total", id="messages-uningested"
+            ),
         ],
     )
     def test_metric_in_output(self, name: str) -> None:

tests/test_uningest.py

@@ -0,0 +1,149 @@
+# Copyright 2026 Logan Fick
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Integration tests for the full ingest → uningest cycle."""
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from crabstero.cache import CachedMessage, IngestCache
+from crabstero.markov import ingest, uningest
+
+if TYPE_CHECKING:
+    import aiosqlite
+
+    from crabstero.database import Database
+
+
+async def _snapshot(db: Database) -> dict[str, list[aiosqlite.Row]]:
+    """Return sorted rows from all Markov-related tables."""
+    tables: dict[str, list[aiosqlite.Row]] = {}
+    for table in ("markov_start_words", "markov_transitions", "channel_images"):
+        async with db._connection.execute(f"SELECT * FROM {table}") as cursor:  # noqa: S608
+            tables[table] = sorted(await cursor.fetchall())
+    return tables
+
+
+class TestIngestUningestCycle:
+    """Full round-trip: ingest data, then uningest it completely."""
+
+    async def test_content_round_trip(self, db: Database) -> None:
+        """Ingest and uningest content leaves the database clean."""
+        await ingest(db, 1, 100, "Hello beautiful world.")
+        await uningest(db, 1, 100, "Hello beautiful world.")
+
+        assert await db.get_random_start_word(1) is None
+        assert await db.get_random_next_word(1, "Hello") is None
+        assert await db.get_random_next_word(1, "beautiful") is None
+
+    async def test_image_round_trip(self, db: Database) -> None:
+        """Ingest and uningest an image leaves the database clean."""
+        await db.add_images([(1, 100, "https://example.com/cat.png")])
+        await db.remove_images([(1, 100, "https://example.com/cat.png")])
+
+        assert await db.get_random_image(1) is None
+
+    async def test_cache_round_trip(self) -> None:
+        """Cache put then pop returns the original entry."""
+        cache = IngestCache()
+        entry = CachedMessage(
+            channel_id=1,
+            user_id=100,
+            content="Hello world.",
+            embed_texts=["Title"],
+            image_urls=["https://example.com/cat.png"],
+        )
+        cache.put(12345, entry)
+        assert cache.pop(12345) is entry
+        assert cache.pop(12345) is None
+
+    async def test_multi_sentence_round_trip(self, db: Database) -> None:
+        """Multi-sentence ingest and uningest leaves the database clean."""
+        text = "Hello world. Goodbye world! How are you?"
+        await ingest(db, 1, 100, text)
+        await uningest(db, 1, 100, text)
+
+        assert await db.get_random_start_word(1) is None
+
+    async def test_uningest_only_removes_one_copy(self, db: Database) -> None:
+        """Uningesting once preserves data from a second identical ingest."""
+        await ingest(db, 1, 100, "Hello world.")
+        await ingest(db, 1, 100, "Hello world.")
+        await uningest(db, 1, 100, "Hello world.")
+
+        # One copy of each row should remain.
+        assert await db.get_random_start_word(1) == "Hello"
+        assert await db.get_random_next_word(1, "Hello") == "world."
+
+    async def test_uningest_does_not_affect_other_channels(self, db: Database) -> None:
+        """Uningesting from one channel leaves another channel's data intact."""
+        await ingest(db, 1, 100, "Hello world.")
+        await ingest(db, 2, 100, "Hello world.")
+        await uningest(db, 1, 100, "Hello world.")
+
+        assert await db.get_random_start_word(1) is None
+        assert await db.get_random_start_word(2) == "Hello"
+        assert await db.get_random_next_word(2, "Hello") == "world."
+
+
+class TestUningestRestoresState:
+    """Uningest restores the database to its prior state."""
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("Hello world.", id="simple-sentence"),
+            pytest.param("Hello world", id="missing-punctuation"),
+            pytest.param(
+                "Hello world. Goodbye world! How are you?",
+                id="multi-sentence",
+            ),
+            pytest.param("One.", id="single-word"),
+            pytest.param(
+                "Lots of extra   spaces\nand\nnewlines   here.",
+                id="whitespace-normalization",
+            ),
+        ],
+    )
+    async def test_uningest_restores_empty_db(self, db: Database, text: str) -> None:
+        """Ingest then uningest on an empty database leaves all tables empty."""
+        before = await _snapshot(db)
+        await ingest(db, 1, 100, text)
+        await uningest(db, 1, 100, text)
+        after = await _snapshot(db)
+        assert after == before
+
+    @pytest.mark.parametrize(
+        "text",
+        [
+            pytest.param("Hello world.", id="simple-sentence"),
+            pytest.param(
+                "Hello world. Goodbye world! How are you?",
+                id="multi-sentence",
+            ),
+        ],
+    )
+    async def test_uningest_restores_preexisting_data(
+        self, db: Database, text: str
+    ) -> None:
+        """Ingest then uningest preserves unrelated pre-existing data exactly."""
+        await ingest(db, 99, 200, "Pre-existing data stays safe.")
+        await db.add_images([(99, 200, "https://example.com/existing.png")])
+
+        before = await _snapshot(db)
+        await ingest(db, 1, 100, text)
+        await uningest(db, 1, 100, text)
+        after = await _snapshot(db)
+        assert after == before