Inference

# inference.py
from openai import OpenAI

# Replace with your actual API key from Settings > API Keys
simplismart_api_key = "YOUR_API_KEY"

# Replace with your endpoint for the Gemma-3-1B model from the Model details page 
endpoint_url = "YOUR_MODEL_ENDPOINT"

# Optional request identifier for tracking in logs
request_id = "e68343f3-7d0b-4b37-af9a-b0ad2c7eefc7"

try:
    # Initialize the OpenAI client with Simplismart endpoint
    client = OpenAI(
        api_key=simplismart_api_key,
        base_url=endpoint_url,        
        default_headers={"id": request_id}
    )
    
    # Define model and prompt
    MODEL_NAME = "gemma-3"
    PROMPT = "What is quantization in GenAI models?"    

    print(f"User: {PROMPT}\n")
    print("AI Assistant: ", end="", flush=True)

    # Create a streaming completion request
    stream = client.chat.completions.create(
        model=MODEL_NAME,
        messages=[
            {
                "role": "system",
                "content": "You are a helpful AI assistant."
            },
            {
                "role": "user", 
                "content": PROMPT
            }
        ],
        max_tokens=512,  # Response length limit
        stream=True,     # Enable streaming for faster first token
    )

    # Process the streamed response
    for chunk in stream:        
        text_delta = chunk.choices[0].delta.content
        if text_delta:            
            print(text_delta, end="", flush=True)
    print()  # Add newline after response

except Exception as e:    
    print(f"An unexpected error occurred: {e}")

pip install openai

python inference.py