library_name: transformers
tags:
- fp8
- qwen
- abliterated
- llmcompressor
⚠️ Disclaimer: FP8 quant of Huihui-Qwen3-8B-abliterated-v2. Safety refusals have been removed (abliterated). User assumes all legal responsibility for its use and output.
quantization
from transformers import AutoTokenizer, AutoModelForCausalLM
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor import oneshot
# Configuration
MODEL_ID = "huihui-ai/Huihui-Qwen3-8B-abliterated-v2"
SAVE_DIR = "Huihui-Qwen3-8B-abliterated-v2-FP8"
# Load model
print(f"Loading model: {MODEL_ID} ...")
tokenizer = AutoTokenizer.from_pretrained(
MODEL_ID,
trust_remote_code=True
)
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
device_map="auto",
torch_dtype="auto",
trust_remote_code=True
)
# Configure quantization scheme: FP8 dynamic quantization, no calibration data needed
recipe = QuantizationModifier(
targets="Linear",
scheme="FP8_DYNAMIC",
ignore=["lm_head"] # Maintain output stability
)
# Execute quantization
print("Starting to apply FP8 quantization (this may take a few minutes)...")
oneshot(
model=model,
recipe=recipe,
)
# Save model
print(f"Quantization complete, saving model to: {SAVE_DIR} ...")
model.save_pretrained(
SAVE_DIR,
save_compressed=True,
max_shard_size="50GB"
)
tokenizer.save_pretrained(SAVE_DIR)
print("Save successful!")
print(f"You can load this model using vLLM: vllm serve {SAVE_DIR} --quantization fp8")
push
from huggingface_hub import login
# Login
login(token=hf_token)
# Use model.push_to_hub() directly
REPO_ID = "YifeiDevs/Huihui-Qwen3-8B-abliterated-v2-FP8"
print(f"Pushing model to {REPO_ID} ...")
model.push_to_hub(
REPO_ID,
private=False,
commit_message="Upload FP8 quantized model",
max_shard_size="50GB"
)
tokenizer.push_to_hub(
REPO_ID,
commit_message="Upload tokenizer"
)
print(f"✅ Push successful!")
print(f"Model link: https://huggingface.co/{REPO_ID}")