Public declaration syntax from forge-rs/crates/forge-embed/src/chunking.rs
Original source SHA-256: fbdc79c82f2cf6ea447a067b584a94d6489119f7999c45f048ac7fbea8662a58
Function bodies and constant values are omitted. This is not the complete implementation.

Source line 48
pub trait TextSplitter: Send + Sync {
    /// Splits the input text into chunks.
    ///
    /// # Arguments
    ///
    /// * `text` - The text to split.
    ///
    /// # Returns
    ///
    /// A vector of non-empty string chunks. Returns an empty vector if the
    /// input is empty.
    fn split(&self, text: &str) -> Vec<String>;
}

Source line 84
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct RecursiveCharacterSplitter {

}

Source line 112
pub fn new(chunk_size: usize, overlap: usize) -> Self;

Source line 149
pub fn with_separators(chunk_size: usize, overlap: usize, separators: Vec<String>) -> Self;

Source line 164
pub fn chunk_size(&self) -> usize;

Source line 169
pub fn overlap(&self) -> usize;

Source line 295
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TokenSplitter {

}

Source line 317
pub fn new(tokens_per_chunk: usize, overlap_tokens: usize) -> Self;

Source line 335
pub fn tokens_per_chunk(&self) -> usize;

Source line 340
pub fn overlap_tokens(&self) -> usize;