diff --git a/configs/gemma/gemma4_31b_it.yaml b/configs/gemma/gemma4_31b_it.yaml new file mode 100644 index 0000000..eaab1c3 --- /dev/null +++ b/configs/gemma/gemma4_31b_it.yaml @@ -0,0 +1,11 @@ +model: google/gemma-4-31b-it +gpu-memory-utilization: 0.95 +max-model-len: 8192 +dtype: auto +trust-remote-code: true +quantization: fp8 +kv-cache-dtype: fp8 +enforce-eager: false +enable-prefix-caching: true +enable-chunked-prefill: true +speculative-config: '{"model":"RedHatAI/gemma-4-31B-it-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}' diff --git a/configs/gpt-oss/gpt_oss_120b.yaml b/configs/gpt-oss/gpt_oss_120b.yaml new file mode 100644 index 0000000..8ef2fbd --- /dev/null +++ b/configs/gpt-oss/gpt_oss_120b.yaml @@ -0,0 +1,9 @@ +model: openai/gpt-oss-120b +gpu-memory-utilization: 0.95 +max-model-len: 8192 +dtype: auto +trust-remote-code: true +enforce-eager: false +enable-prefix-caching: true +enable-chunked-prefill: true +speculative-config: '{"model":"RedHatAI/gpt-oss-120b-speculator.eagle3","method":"eagle3","num_speculative_tokens":3}'