import wallaroo
from wallaroo.framework import Framework
from wallaroo.engine_config import Acceleration
from wallaroo.openai_config import OpenaiConfig
import pyarrow as pa

wl = wallaroo.Client(request_timeout=600)

workspace = wl.get_workspace(name='vllm-openai-test', create_if_not_exist=True)
wl.set_current_workspace(workspace)

{'name': 'vllm-openai-test', 'id': 1689, 'archived': False, 'created_by': 'sample.user@wallaroo.ai', 'created_at': '2025-05-30T20:30:35.093295+00:00', 'models': [{'name': 'tinyllamaopenai', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 5, 30, 20, 32, 28, 757011, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 5, 30, 20, 32, 28, 757011, tzinfo=tzutc())}, {'name': 'tinyllamaopenaiyns1', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 3, 0, 31, 49, 205332, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 3, 0, 31, 49, 205332, tzinfo=tzutc())}, {'name': 'tinyllama', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 3, 0, 34, 0, 798254, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 3, 0, 34, 0, 798254, tzinfo=tzutc())}, {'name': 'ragstep1', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 3, 1, 46, 47, 430142, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 3, 1, 46, 47, 430142, tzinfo=tzutc())}, {'name': 'tinyllamaopenaiyns2', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 16, 16, 36, 23, 762501, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 16, 16, 36, 23, 762501, tzinfo=tzutc())}, {'name': 'tinyllamaopenaiyns-error', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 23, 15, 2, 59, 581760, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 23, 15, 2, 59, 581760, tzinfo=tzutc())}, {'name': 'tinyllamaopenaiyns-error1', 'versions': 1, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 24, 18, 39, 42, 466411, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 24, 18, 39, 42, 466411, tzinfo=tzutc())}, {'name': 'tinyllamarag', 'versions': 2, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 27, 17, 43, 44, 446012, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 3, 19, 25, 43, 437726, tzinfo=tzutc())}, {'name': 'ragstep', 'versions': 3, 'owner_id': '""', 'last_update_time': datetime.datetime(2025, 6, 27, 17, 49, 39, 238424, tzinfo=tzutc()), 'created_at': datetime.datetime(2025, 6, 3, 0, 37, 17, 945954, tzinfo=tzutc())}], 'pipelines': [{'name': 'tinyllama-openai', 'create_time': datetime.datetime(2025, 5, 30, 20, 40, 46, 518566, tzinfo=tzutc()), 'definition': '[]'}, {'name': 'tinyllama-openai-error', 'create_time': datetime.datetime(2025, 6, 23, 15, 13, 57, 625524, tzinfo=tzutc()), 'definition': '[]'}, {'name': 'tinyllama-openai-error1', 'create_time': datetime.datetime(2025, 6, 24, 18, 43, 42, 785405, tzinfo=tzutc()), 'definition': '[]'}, {'name': 'tinyllama-openai-rag-cb', 'create_time': datetime.datetime(2025, 6, 4, 18, 15, 37, 345076, tzinfo=tzutc()), 'definition': '[]'}, {'name': 'tinyllama-openai-rag', 'create_time': datetime.datetime(2025, 6, 3, 0, 43, 13, 169150, tzinfo=tzutc()), 'definition': '[]'}]}

model_step = wl.upload_model(
    "tinyllamaopenaiyns1",
    "vllm-openai_tinyllama.zip",
    framework=Framework.VLLM,
    input_schema=pa.schema([]),
    output_schema=pa.schema([]),
    convert_wait=True,
    accel=Acceleration.CUDA
)

Waiting for model loading - this will take up to 10min.

Model is pending loading to a container runtime...........................
Model is attempting loading to a container runtime...................................
Successful
Ready

# Configuring as OpenAI

openai_config = OpenaiConfig(enabled=True, chat_completion_config={"temperature": .3, "max_tokens": 200})
model_step = model_step.configure(openai_config=openai_config)

# Deploying

deployment_config = wallaroo.DeploymentConfigBuilder() \
    .replica_count(1) \
    .cpus(.5) \
    .memory("1Gi") \
    .sidekick_cpus(model_step, 1) \
    .sidekick_memory(model_step, '8Gi') \
    .sidekick_gpus(model_step, 1) \
    .deployment_label('wallaroo.ai/accelerator:l4') \
    .build()

pipeline = wl.build_pipeline('tinyllama-openai')
pipeline.clear()
pipeline.add_model_step(model_step)
pipeline.deploy(deployment_config = deployment_config)

# Performing completions

pipeline.openai_chat_completion(messages=[{"role": "user", "content": "good morning"}]).choices[0].message.content

"Of course! Here's an updated version of the text with the added phrases:\n\nAs the sun rises over the horizon, the world awakens to a new day. The birds chirp and the birdsong fills the air, signaling the start of another beautiful day. The gentle breeze carries the scent of freshly cut grass and the promise of a new day ahead. The sun's rays warm the skin, casting a golden glow over everything in sight. The world awakens to a new day, a new chapter, a new beginning. The world is alive with energy and vitality, ready to take on the challenges of the day ahead. The birds chirp and the birdsong fills the air, signaling the start of another beautiful day. The gentle breeze carries the scent of freshly cut grass and the promise of a new day ahead. The sun's rays warm the skin"

pipeline.openai_completion(prompt="tell me about wallaroo.AI", max_tokens=200).choices[0].text

', any first-person shooter game. Wallaroo is a comprehensive platform for building and tracking predictive models. This tool is really helpful in AI development. Wallaroo provides a unified platform for data and model developers to securely store or share data and access/optimize their AI models. It allows end-users to have a direct access to the development tools to customize and reuse code. Wallaroo has an intuitive User Interface that is easy to install and configure. Wallaroo handles entire the integration, deployment and infrastructure from data collection to dashboard visualisations. Can you provide some examples of how Wallaroo has been utilised in game development? Also, talk about the effectiveness of ML training using Wallaroo.'

# Now with streaming

for chunk in pipeline.openai_chat_completion(messages=[{"role": "user", "content": "this is a short story about love"}], max_tokens=100, stream=True):
    print(chunk.choices[0].delta.content, end="", flush=True)

Once upon a time, in a small village nestled in the heart of the countryside, there lived a young woman named Lily. Lily was a kind and gentle soul, always looking out for those in need. She had a heart full of love for her family and friends, and she was always willing to lend a helping hand.

One day, Lily met a handsome young man named Jack. Jack was a charming and handsome man, with a

# Now with streaming

for chunk in pipeline.openai_completion(prompt="tell me a short story", max_tokens=300, stream=True):
    print(chunk.choices[0].text, end="", flush=True)

?" this makes their life easier, but sometimes, when they have a story, they don't know how to tell it well. This frustrates them and makes their life even more difficult.

b. Relaxation:
protagonist: take a deep breath and let it out. Why not start with a song? "Eyes full of longing, I need your music to embrace." this calms them down and lets them relax, giving them more patience to continue with their story.

c. Inspirational quotes:
protagonist: this quote from might jeffries helps me reflect on my beliefs and values: "the mind is a powerful thing, it can change your destiny at any time. Fear no fear, only trust your divineline and reclaim your destiny." listening to this quote always helps me keep my thoughts in perspective, and gets me back to my story with renewed vigor.

# Now using the OpenAI client

token = wl.auth.auth_header()['Authorization'].split()[1]

from openai import OpenAI
client = OpenAI(
    base_url='https://example.wallaroo.ai/v1/api/pipelines/infer/tinyllama-openai-414/tinyllama-openai/openai/v1',
    api_key=token
)

for chunk in client.chat.completions.create(model="dummy", 
                                            messages=[{"role": "user", "content": "this is a short story about love"}], 
                                            max_tokens=1000, 
                                            stream=True):
    print(chunk.choices[0].delta.content, end="", flush=True)

It was a warm summer evening, and the sun was setting over the city. A young couple, Alex and Emily, had just walked out of a coffee shop, hand in hand. They were laughing and chatting, enjoying the last few moments of their day.

As they walked down the street, Alex turned to Emily and said, "I love you, Emily."

Emily's eyes widened in surprise, and she smiled. "I love you too, Alex."

They walked for a few more blocks, hand in hand, and finally, they arrived at a park. They sat down on a bench, and Alex took Emily's hand.

"I know this is a little sudden," Alex said, "but I feel like we've been together for a while now. I want to spend the rest of my life with you."

Emily looked at him, her eyes filled with tears. "I feel the same way, Alex. I love you more than anything in this world."

They sat there, holding hands, for what felt like hours. They talked about everything and anything, their hearts beating in unison.

As the sun began to set, Alex and Emily stood up, and they walked back to the coffee shop. They hugged each other tightly, tears streaming down their faces.

"I love you, Alex," Emily said, her voice shaking.

Alex smiled, and he said, "I love you too, Emily."

They walked back to their apartment, hand in hand, and spent the rest of the night talking and laughing.

Over the next few weeks, Alex and Emily's love grew stronger. They spent every moment they could together, exploring the city, going on walks, and enjoying each other's company.

One day, they decided to take a walk in a nearby park. As they walked, they talked about everything and anything, their hearts beating in unison.

As they reached the end of the path, Alex turned to Emily and said, "I love you, Emily. I know we've only been together for a short time, but I feel like we've been through so much together. I want to spend the rest of my life with you."

Emily looked at him, her eyes filled with tears. "I feel the same way, Alex. I love you more than anything in this world."

They stood there, holding hands, and Alex said, "I love you, Emily. I want to spend the rest of my life with you."

Emily smiled, tears streaming down her face. "I love you too, Alex. I know this is a little sudden, but I feel like we've been together for a while now. I want to spend the rest of my life with you."

They walked back to their apartment, hand in hand, and spent the rest of the night talking and laughing.

From that day on, Alex and Emily's love grew stronger. They spent every moment they could together, exploring the city, going on walks, and enjoying each other's company.

Years later, they were married, and they had two beautiful children. They knew that their love had been a little sudden, but they knew that it was worth it. They knew that they had found each other, and they knew that they would spend the rest of their lives together, loving and cherishing each other.

for chunk in client.completions.create(model="dummy", prompt="tell me about wallaroo.AI", max_tokens=1000, stream=True):
    print(chunk.choices[0].text, end="", flush=True)

's robotic fabrication technology: can you provide some examples of products that have been milled using wallaroo’s robots?

pipeline.publish(deployment_config)

Waiting for pipeline publish... It may take up to 600 sec.
................................. Published.

pipeline.undeploy()

Waiting for undeployment - this will take up to 600s .................................... ok