Inference

# inference.py
from openai import OpenAI

# Replace with your actual API key from Settings > API Keys
simplismart_api_key = "SIMPLISMART_API_KEY"    

# Replace with your endpoint for the Gemma-3-1B model from the Model details page 
simplismart_base_url = "https://api.simplismart.live"

try:
    # Initialize the OpenAI client with Simplismart endpoint
    client = OpenAI(
        api_key=simplismart_api_key,
        base_url=simplismart_base_url,        
    )
    
    # Define model and prompt
    MODEL_NAME = "google/gemma-4-31B-it"
    PROMPT = "What is quantization in GenAI models?"    

    print(f"User: {PROMPT}\n")
    print("AI Assistant: ", end="", flush=True)

    # Create a streaming completion request
    stream = client.chat.completions.create(
        model=MODEL_NAME,
        messages=[
            {
                "role": "system",
                "content": "You are a helpful AI assistant."
            },
            {
                "role": "user", 
                "content": PROMPT
            }
        ],
        max_tokens=512,  # Response length limit
        stream=True,     # Enable streaming for faster first token
    )

    # Process the streamed response
    for chunk in stream:        
        text_delta = chunk.choices[0].delta.content
        if text_delta:            
            print(text_delta, end="", flush=True)
    print()  # Add newline after response

except Exception as e:    
    print(f"An unexpected error occurred: {e}")

pip install openai

python inference.py