LogalDeveloper/Crabstero
LogalDeveloper
/
Crabstero
0
Fork 0
Code Issues 1 Actions Packages Releases Wiki Activity

Reverted "Reworked organization of Redis database logic." 

Turns out Apache Kvrocks does not implement logical databases the same way as Redis. The SELECT command will accept any number and pretend to switch logical databases, but in reality it does not seem to do anything. Because of this, the original database logic is better suited. If a non-Redis database is used, such as Apache Kvrocks, the new database logic is actually a potential regression.
This commit is contained in:
Logan Fick 2023-04-29 20:52:52 -04:00
parent 4a22c1abfc
commit ae259a5c1b
Signed by: LogalDeveloper
GPG Key ID: 43E58A0C922AB7D1
5 changed files with 126 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/main/java/dev/logal/crabstero/Crabstero.java
View File

@ -1,5 +1,5 @@
/*
* Copyright 2023 Logan Fick
* Copyright 2022 Logan Fick
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
*
@ -9,12 +9,13 @@
package dev.logal.crabstero;
import dev.logal.crabstero.listeners.*;
import dev.logal.crabstero.utils.RedisDatabase;
import org.javacord.api.DiscordApi;
import org.javacord.api.DiscordApiBuilder;
import org.javacord.api.entity.intent.Intent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -26,7 +27,7 @@ public final class Crabstero {
private static final Logger logger = LoggerFactory.getLogger(Crabstero.class);
private final DiscordApi discordApi;
private final RedisDatabase redisDatabase;
private final JedisPool jedisPool;
private final ScheduledExecutorService workerPool;
/**
@ -36,12 +37,8 @@ public final class Crabstero {
* @param redisHost The host of the Redis server.
* @param redisPort The port of the Redis server.
*/
public Crabstero(final String token, final String redisHost, final Integer redisPort) {
if (redisPort == null) {
this.redisDatabase = new RedisDatabase(redisHost);
} else {
this.redisDatabase = new RedisDatabase(redisHost, redisPort);
}
public Crabstero(final String token, final String redisHost, final int redisPort) {
this.jedisPool = new JedisPool(new JedisPoolConfig(), redisHost, redisPort);
this.workerPool = Executors.newScheduledThreadPool(4, new CrabsteroThreadFactory());
final DiscordApiBuilder builder = new DiscordApiBuilder();
@ -75,14 +72,10 @@ public final class Crabstero {
public static void main(final String[] arguments) {
final String token = System.getenv("TOKEN");
final String redisHost = System.getenv("REDIS_HOST");
final String redisPort = System.getenv("REDIS_PORT");
final int redisPort = Integer.parseInt(System.getenv("REDIS_PORT"));
logger.info("Starting Crabstero...");
if (redisPort == null){
new Crabstero(token, redisHost, null);
} else {
new Crabstero(token, redisHost, Integer.parseInt(redisPort));
}
new Crabstero(token, redisHost, redisPort);
logger.info("Crabstero started!");
}
@ -96,12 +89,12 @@ public final class Crabstero {
}
/**
* Gets the Redis database used by this instance of Crabstero.
* Gets the Jedis pool used by this instance of Crabstero.
*
* @return The Redis database.
* @return The Jedis pool.
*/
public RedisDatabase getRedisDatabase() {
return this.redisDatabase;
public JedisPool getJedisPool() {
return this.jedisPool;
}
/**

25
src/main/java/dev/logal/crabstero/tasks/ChannelHistoryIngestionTask.java
View File

@ -1,5 +1,5 @@
/*
* Copyright 2023 Logan Fick
* Copyright 2022 Logan Fick
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
*
@ -10,11 +10,11 @@ package dev.logal.crabstero.tasks;
import dev.logal.crabstero.Crabstero;
import dev.logal.crabstero.utils.MarkovChainMessages;
import dev.logal.crabstero.utils.RedisDatabase;
import org.javacord.api.entity.channel.ServerTextChannel;
import org.javacord.api.entity.message.Message;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import redis.clients.jedis.Jedis;
import java.util.Iterator;
import java.util.stream.Stream;
@ -55,17 +55,18 @@ public final class ChannelHistoryIngestionTask implements Runnable {
return;
}
try (final Jedis jedis = this.crabstero.getJedisPool().getResource()) {
// Have we already ingested this channel before?
if (jedis.lrange(INGESTED_CHANNELS_KEY, 0, -1).contains(this.channel.getIdAsString())) {
// Yes. Stop further processing of this channel.
return;
} else {
// No. Add this channel's ID to the list of ingested channels.
jedis.lpush(INGESTED_CHANNELS_KEY, this.channel.getIdAsString());
// Have we already ingested this channel before?
if (this.crabstero.getRedisDatabase().lrange(RedisDatabase.MAIN_DATABASE_ID, 0, INGESTED_CHANNELS_KEY, 0, -1).contains(this.channel.getIdAsString())) {
// Yes. Stop further processing of this channel.
return;
} else {
// No. Add this channel's ID to the list of ingested channels.
this.crabstero.getRedisDatabase().lpush(RedisDatabase.MAIN_DATABASE_ID, 0, INGESTED_CHANNELS_KEY, this.channel.getIdAsString());
// Log the start of the ingestion of this channel.
logger.info("Starting ingestion of text channel history. (Name: \"" + this.channel.getName() + "\" | ID: " + this.channel.getIdAsString() + " | Server ID: " + this.channel.getServer().getIdAsString() + ")");
// Log the start of the ingestion of this channel.
logger.info("Starting ingestion of text channel history. (Name: \"" + this.channel.getName() + "\" | ID: " + this.channel.getIdAsString() + " | Server ID: " + this.channel.getServer().getIdAsString() + ")");
}
}
final MarkovChainMessages markovChainMessages = new MarkovChainMessages(this.crabstero);

171
src/main/java/dev/logal/crabstero/utils/MarkovChain.java
View File

@ -1,5 +1,5 @@
/*
* Copyright 2023 Logan Fick
* Copyright 2022 Logan Fick
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
*
@ -9,6 +9,8 @@
package dev.logal.crabstero.utils;
import dev.logal.crabstero.Crabstero;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.Pipeline;
import java.security.SecureRandom;
import java.util.List;
@ -18,11 +20,10 @@ import java.util.Random;
* Ingests sentences and generates new ones using a Markov chain with a Redis storage.
*/
public final class MarkovChain {
private static final String STARTING_WORDS_KEY = "startingWords";
private static final char DEFAULT_SENTENCE_END = '§'; // Default character to end sentences with if it does not include a period, exclamation mark, or question mark.
private final long id; // A unique number for this Markov chain to segment it away from other Markov chains. This is usually the channel ID given by Discord.
private final RedisDatabase redisDatabase; // The redis database for the Markov chain to store and retrieve data from.
private final Crabstero crabstero; // The Crabstero instance using this Markov chain.
private final Random rng; // A source of randomness.
/**
@ -33,7 +34,7 @@ public final class MarkovChain {
*/
public MarkovChain(final long id, final Crabstero crabstero) {
this.id = id;
this.redisDatabase = crabstero.getRedisDatabase();
this.crabstero = crabstero;
this.rng = new SecureRandom(); // Probably placebo effect, but SecureRandom seems to produce better results.
}
@ -46,7 +47,7 @@ public final class MarkovChain {
*/
public MarkovChain(final long id, final Crabstero crabstero, final Random random) {
this.id = id;
this.redisDatabase = crabstero.getRedisDatabase();
this.crabstero = crabstero;
this.rng = random;
}
@ -111,16 +112,22 @@ public final class MarkovChain {
// Finally, split the sentence into a string array with each element being a word.
String[] words = sentence.trim().replaceAll(" +", " ").split(" ");
// Iterate over the array of words, starting from the first element until the second to last element.
for (int i = 0; i < words.length - 1; i++) {
// Is this the first word?
if (i == 0) {
// Yes. Push the word into a special list containing starting words.
this.redisDatabase.lpush(RedisDatabase.MAIN_DATABASE_ID, this.id, STARTING_WORDS_KEY, words[i]);
}
// Get a reference to Jedis from the pool.
try (final Jedis jedis = this.crabstero.getJedisPool().getResource()) {
// Use a pipeline to avoid blocking caused by waiting for responses from the Redis server.
final Pipeline pipeline = jedis.pipelined();
// Push the word after this one into a list identified by this word.
this.redisDatabase.lpush(RedisDatabase.WORDS_DATABASE_ID, this.id, words[i], words[i + 1]);
// Iterate over the array of words, starting from the first element until the second to last element.
for (int i = 0; i < words.length - 1; i++) {
// Is this the first word?
if (i == 0) {
// Yes. Push the word into a special list containing starting words.
pipeline.lpush(this.id + ":start", words[i]);
}
// Push the word after this one into a list identified by this word.
pipeline.lpush(this.id + "::" + words[i], words[i + 1]);
}
}
}
@ -135,76 +142,80 @@ public final class MarkovChain {
// Declare a new builder for building the new sentence.
final StringBuilder newSentence = new StringBuilder();
// Does this Markov chain have any starting words?
if (!this.redisDatabase.exists(RedisDatabase.MAIN_DATABASE_ID, this.id, STARTING_WORDS_KEY)) {
// No. Quickly ingest something to avoid throwing an error.
this.ingestSentence("Hello world!");
}
// Declare a variable for holding the word about to be added to the new sentence.
String word;
// Get a list of all starting words.
final List<String> startingWords = this.redisDatabase.lrange(RedisDatabase.MAIN_DATABASE_ID, this.id, STARTING_WORDS_KEY, 0, 499); // TODO: What's a good stop value? 0 seems like a potential DoS vector.
// Pick a random index for a starting word.
int index = rng.nextInt(startingWords.size());
// Get the word.
word = startingWords.get(index);
// Add it to the output sentence.
newSentence.append(word);
// While the selected word is in complete...
// Although quite rare, this loop will skip if the starting word is already complete on its own (e.g. "Yes.")
while (!isCompleteSentence(word)) {
// Get the list of words which can be added after the previous word.
final List<String> wordChoices = this.redisDatabase.lrange(RedisDatabase.WORDS_DATABASE_ID, this.id, word, 0, 499); // TODO: What's a good stop value? 0 seems like a potential DoS vector.
// Reset the index.
index = -1;
// Is the new sentence length above the soft character limit?
if (newSentence.length() >= softCharacterLimit) {
// Yes. Time to aggressively search for words which can complete this sentence.
// Iterate over the word choices.
for (int i = 0; i < wordChoices.size(); i++) {
// Get current candidate word.
final String candidate = wordChoices.get(i);
// Does this word conclude the sentence?
if (isCompleteSentence(candidate)) {
// Select the index for this word and stop evaluating choices.
index = i;
break;
}
}
// Was no word to conclude the new sentence found?
if (index == -1) {
// Yes. Pick a random word and hope the next word can conclude it.
index = rng.nextInt(wordChoices.size());
}
} else {
// No, there is still room. Pick a random index.
index = rng.nextInt(wordChoices.size());
// Get a reference to Jedis from the pool.
try (final Jedis jedis = this.crabstero.getJedisPool().getResource()) {
// Does this Markov chain have any starting words?
if (!jedis.exists(this.id + ":start")) {
// No. Quickly ingest something to avoid throwing an error.
// TODO: Could this soft lock since a reference to the jedis pool is already held?
this.ingestSentence("Hello world!");
}
// Get the selected word.
word = wordChoices.get(index);
// Declare a variable for holding the word about to be added to the new sentence.
String word;
// Append a space and this word to the new sentence.
newSentence.append(" ").append(word);
// Get a list of all starting words.
final List<String> startingWords = jedis.lrange(this.id + ":start", 0, -1);
// Is the new sentence now longer than the hard character limit?
final int sentenceLength = newSentence.length();
if (sentenceLength >= hardCharacterLimit) {
// Yes. Chop off the characters beyond the hard limit.
newSentence.delete(hardCharacterLimit, sentenceLength);
// Break out of the loop to forcefully declare the new sentence complete.
break;
// Pick a random index for a starting word.
int index = rng.nextInt(startingWords.size());
// Get the word.
word = startingWords.get(index);
// Add it to the output sentence.
newSentence.append(word);
// While the selected word is in complete...
// Although quite rare, this loop will skip if the starting word is already complete on its own (e.g. "Yes.")
while (!isCompleteSentence(word)) {
// Get the list of words which can be added after the previous word.
final List<String> wordChoices = jedis.lrange(this.id + "::" + word, 0, -1);
// Reset the index.
index = -1;
// Is the new sentence length above the soft character limit?
if (newSentence.length() >= softCharacterLimit) {
// Yes. Time to aggressively search for words which can complete this sentence.
// Iterate over the word choices.
for (int i = 0; i < wordChoices.size(); i++) {
// Get current candidate word.
final String candidate = wordChoices.get(i);
// Does this word conclude the sentence?
if (isCompleteSentence(candidate)) {
// Select the index for this word and stop evaluating choices.
index = i;
break;
}
}
// Was no word to conclude the new sentence found?
if (index == -1) {
// Yes. Pick a random word and hope the next word can conclude it.
index = rng.nextInt(wordChoices.size());
}
} else {
// No, there is still room. Pick a random index.
index = rng.nextInt(wordChoices.size());
}
// Get the selected word.
word = wordChoices.get(index);
// Append a space and this word to the new sentence.
newSentence.append(" ").append(word);
// Is the new sentence now longer than the hard character limit?
final int sentenceLength = newSentence.length();
if (sentenceLength >= hardCharacterLimit) {
// Yes. Chop off the characters beyond the hard limit.
newSentence.delete(hardCharacterLimit, sentenceLength);
// Break out of the loop to forcefully declare the new sentence complete.
break;
}
}
}

17
src/main/java/dev/logal/crabstero/utils/MarkovChainMessages.java
View File

@ -1,5 +1,5 @@
/*
* Copyright 2023 Logan Fick
* Copyright 2022 Logan Fick
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
*
@ -18,6 +18,7 @@ import org.javacord.api.entity.message.embed.EmbedBuilder;
import org.javacord.api.entity.message.mention.AllowedMentions;
import org.javacord.api.entity.message.mention.AllowedMentionsBuilder;
import org.javacord.api.util.logging.ExceptionLogger;
import redis.clients.jedis.Jedis;
import java.security.SecureRandom;
import java.util.List;
@ -27,8 +28,6 @@ import java.util.Random;
* Assists with generating Discord messages in response to other users and ingesting raw messages.
*/
public class MarkovChainMessages {
private static final String IMAGE_URLS_KEY = "imageURLS";
private final Crabstero crabstero;
private final AllowedMentions allowedMentions;
private final Random rng = new SecureRandom();
@ -86,10 +85,12 @@ public class MarkovChainMessages {
embed.setDescription(markovChain.generate(300, 500));
// If image URLs are known for this channel, chose a random one and attach to the embed.
final List<String> embedImageURLs = this.crabstero.getRedisDatabase().lrange(RedisDatabase.MAIN_DATABASE_ID, channelID, IMAGE_URLS_KEY, 0, 499); // TODO: What's a good stop value? 0 seems like a potential DoS vector.
try (final Jedis jedis = this.crabstero.getJedisPool().getResource()) {
final List<String> embedImageURLs = jedis.lrange(channelID + ":images", 0, -1);
if (embedImageURLs.size() > 0) {
embed.setImage(embedImageURLs.get(this.rng.nextInt(embedImageURLs.size())));
if (embedImageURLs.size() > 0) {
embed.setImage(embedImageURLs.get(this.rng.nextInt(embedImageURLs.size())));
}
}
// Add a watermark to the footer of the embed.
@ -148,7 +149,9 @@ public class MarkovChainMessages {
// If the embed has an image, store the URL to the image.
embed.getImage().ifPresent((image) -> {
this.crabstero.getRedisDatabase().lpush(RedisDatabase.MAIN_DATABASE_ID, channelID, IMAGE_URLS_KEY, image.getUrl().toString());
try (final Jedis jedis = this.crabstero.getJedisPool().getResource()) {
jedis.lpush(channelID + ":images", image.getUrl().toString());
}
});
}
}

110
src/main/java/dev/logal/crabstero/utils/RedisDatabase.java
View File

@ -1,110 +0,0 @@
/*
* Copyright 2023 Logan Fick
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at: https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
*/
package dev.logal.crabstero.utils;
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.JedisPoolConfig;
import java.util.List;
/**
* Wrapper for a Redis database which forces logical separation of data by requiring two extra parameters for every command:
* 1. A logical database ID, which is intended to separate Markov chain data from other miscellaneous state data.
* 2. A general unique ID, which is intended to separate data in the same logical database between Discord channels.
*/
public final class RedisDatabase {
public static final int MAIN_DATABASE_ID = 0; // The ID of the logical database which stores miscellaneous state data.
public static final int WORDS_DATABASE_ID = 1; // The ID of the logical database which specifically stores Markov chain data.
private static final int DEFAULT_REDIS_PORT = 6379; // Default port to use when not specifically during instantiation.
private static final String SEPARATOR = ":"; // Separator to use between the virtual namespace ID and key.
private final JedisPool jedisPool;
/**
* Creates a new Redis database which connects to the Redis server hosted on the given host and port DEFAULT_REDIS_PORT.
*
* @param redisHost The host of the Redis server.
*/
public RedisDatabase(final String redisHost) {
this.jedisPool = new JedisPool(new JedisPoolConfig(), redisHost, DEFAULT_REDIS_PORT);
}
/**
* Creates a new Redis database which connects to the Redis server hosted on the given host and port.
*
* @param redisHost The host of the Redis server.
* @param redisPort The port of the Redis server.
*/
public RedisDatabase(final String redisHost, final int redisPort) {
this.jedisPool = new JedisPool(new JedisPoolConfig(), redisHost, redisPort);
}
/**
* Converts a given unique ID and a key into a single String which can be used as a key to the backend Redis database.
*
* @param id The unique ID.
* @param key The key.
* @return The long key which can be sent to a Redis database, unless the given unique ID is 0, then just key exactly as given.
*/
private static String resolveKey(final long id, final String key) {
if (id == 0) {
return key;
} else {
return id + SEPARATOR + key;
}
}
/**
* Adds a string to the head of the list stored at the given key uniquely identified by the given ID in the given logical database ID.
*
* @param database Zero-based index logical database ID.
* @param id The unique ID.
* @param key The key of the list.
* @param values The value(s) to add to the list.
*/
public void lpush(final int database, final long id, final String key, final String... values) {
try (final Jedis jedis = this.jedisPool.getResource()) {
jedis.select(database);
jedis.lpush(resolveKey(id, key), values);
}
}
/**
* Tests whether the given key uniquely identified by the given ID in the given logical database ID exists.
*
* @param database Zero-based index logical database ID.
* @param id The unique ID.
* @param key The key to test.
* @return True if the key exists, false otherwise.
*/
public boolean exists(final int database, final long id, final String key) {
try (final Jedis jedis = this.jedisPool.getResource()) {
jedis.select(database);
return jedis.exists(resolveKey(id, key));
}
}
/**
* Returns the elements starting and ending at the given indexes of the list stored at the given key uniquely identified by the given ID in the given logical database ID.
*
* @param database Zero-based index logical database ID.
* @param id The unique ID
* @param key The key of the list.
* @param start Zero-based index of the starting element.
* @param stop Zero-based index of the ending element.
* @return A List with the requested elements.
*/
public List<String> lrange(final int database, final long id, final String key, final long start, final long stop) {
try (final Jedis jedis = this.jedisPool.getResource()) {
jedis.select(database);
return jedis.lrange(resolveKey(id, key), start, stop);
}
}
}