Struct BPE 

pub struct BPE {
    pub re: Regex,
    pub end_of_text: usize,
    pub offset: usize,
    pub ranks: HashMap<Vec<u8>, u32>,
    /* private fields */
}

Fields

re: Regex

end_of_text: usize

offset: usize

ranks: HashMap<Vec<u8>, u32>

Implementations

impl BPE

pub fn from_json(json: &Value, end_of_text: usize) -> Result<Self>

pub fn byte_pair_encode(&self, piece: &[u8]) -> Vec<u32>

pub fn encode(&self, text: &str) -> Result<Vec<u32>>