29 lines
745 B
Rust
29 lines
745 B
Rust
#[derive(Clone, Debug)]
|
|
pub struct SmolLMConfig {
|
|
pub hidden_size: usize,
|
|
pub intermediate_size: usize,
|
|
pub vocab_size: usize,
|
|
pub num_hidden_layers: usize,
|
|
pub num_attention_heads: usize,
|
|
pub num_key_value_heads: usize,
|
|
pub rms_norm_eps: f64,
|
|
pub rope_theta: f32,
|
|
pub max_position_embeddings: usize,
|
|
}
|
|
|
|
impl Default for SmolLMConfig {
|
|
fn default() -> Self {
|
|
Self {
|
|
hidden_size: 576,
|
|
intermediate_size: 1536,
|
|
vocab_size: 49152,
|
|
num_hidden_layers: 30,
|
|
num_attention_heads: 9,
|
|
num_key_value_heads: 3,
|
|
rms_norm_eps: 1e-5,
|
|
rope_theta: 10000.0,
|
|
max_position_embeddings: 2048,
|
|
}
|
|
}
|
|
}
|