Together AI SDK Patterns

Overview

Production-ready patterns for Together AI inference. Together exposes an OpenAI-compatible REST API at https://api.together.xyz/v1, meaning any OpenAI client library works with a base URL swap. This makes Together a drop-in replacement for OpenAI when running open-source models (Llama, Mixtral, Qwen, FLUX). A singleton client centralizes the base URL override and enables seamless backend switching.

Singleton Client

import OpenAI from 'openai';
let _client: OpenAI | null = null;
export function getClient(): OpenAI {
  if (!_client) {
    const apiKey = process.env.TOGETHER_API_KEY;
    if (!apiKey) throw new Error('TOGETHER_API_KEY must be set — get it from api.together.xyz/settings');
    _client = new OpenAI({ apiKey, baseURL: 'https://api.together.xyz/v1' });
  }
  return _client;
}
// Usage: const client = getClient();
// await client.chat.completions.create({ model: 'meta-llama/Meta-Llama-3.1-70B-Instruct', messages: [...] });

Error Wrapper

export class TogetherError extends Error {
  constructor(public status: number, public code: string, message: string) { super(message); }
}
export async function safeCall<T>(operation: string, fn: () => Promise<T>): Promise<T> {
  try { return await fn(); }
  catch (err: any) {
    const status = err.status ?? err.response?.status ?? 0;
    if (status === 429) { await new Promise(r => setTimeout(r, 3000)); return fn(); }
    if (status === 401) throw new TogetherError(401, 'AUTH', 'Invalid TOGETHER_API_KEY');
    if (status === 404) throw new TogetherError(404, 'MODEL', `${operation}: model not found — use client.models.list()`);
    throw new TogetherError(status, 'API_ERROR', `${operation} failed [${status}]: ${err.message}`);
  }
}

Request Builder

class TogetherRequest {
  private params: Record<string, any> = {};
  model(m: string) { this.params.model = m; return this; }
  messages(msgs: Array<{ role: string; content: string }>) { this.params.messages = msgs; return this; }
  temperature(t: number) { this.params.temperature = t; return this; }
  maxTokens(n: number) { this.params.max_tokens = n; return this; }
  stream(s = true) { this.params.stream = s; return this; }
  jsonMode() { this.params.response_format = { type: 'json_object' }; return this; }
  build() { return this.params; }
}
// Usage: new TogetherRequest().model('meta-llama/Meta-Llama-3.1-70B-Instruct')
//   .messages([{ role: 'user', content: 'Summarize this' }]).temperature(0.3).jsonMode().build();

Response Types

interface TogetherModel {
  id: string; type: 'chat' | 'language' | 'image' | 'embed

Together AI Security Basics

Overview

Together AI provides inference and fine-tuning for 100+ open-source models (Llama, Mixtral, Qwen, FLUX) via an OpenAI-compatible API. Security concerns include API key management for production inference, protecting fine-tuning datasets that may contain proprietary or sensitive data, rate limit handling to prevent cost overruns, and ensuring model outputs are not logged with sensitive prompt content. A leaked API key grants full access to inference, fine-tuning, and model management endpoints.

API Key Management

function createTogetherClient(): { apiKey: string; baseUrl: string } {
  const apiKey = process.env.TOGETHER_API_KEY;
  if (!apiKey) {
    throw new Error("Missing TOGETHER_API_KEY — store in secrets manager, never in code");
  }
  // Together keys access inference + fine-tuning — treat as production credentials
  console.log("Together AI client initialized (key suffix:", apiKey.slice(-4), ")");
  return { apiKey, baseUrl: "https://api.together.xyz/v1" };
}

Webhook Signature Verification

import crypto from "crypto";
import { Request, Response, NextFunction } from "express";

function verifyTogetherWebhook(req: Request, res: Response, next: NextFunction): void {
  const signature = req.headers["x-together-signature"] as string;
  const secret = process.env.TOGETHER_WEBHOOK_SECRET!;
  const expected = crypto.createHmac("sha256", secret).update(req.body).digest("hex");
  if (!signature || !crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) {
    res.status(401).send("Invalid signature");
    return;
  }
  next();
}

Input Validation

import { z } from "zod";

const InferenceRequestSchema = z.object({
  model: z.string().min(1).max(200),
  messages: z.array(z.object({
    role: z.enum(["system", "user", "assistant"]),
    content: z.string().max(100_000),
  })).min(1),
  max_tokens: z.number().int().min(1).max(4096).default(512),
  temperature: z.number().min(0).max(2).default(0.7),
  stop: z.array(z.string()).max(4).optional(),
});

function validateInferenceRequest(data: unknown) {
  return InferenceRequestSchema.parse(data);
}

Data Protection

const TOGETHER_SENSITIVE_FIELDS = ["api_key", "prompt_content", "fine_tune_dataset", "model_output", "system_prompt"];

function redactTogetherLog(record: Record<string, unknown>): Record<string, unknown> {
  const redacted = { ...record };
  for (const field of TOGETHER_SENSITIVE_FIELDS) {
    if (field in redacted) redacted[field] = "[REDACTED]";
  }
  return redacted;
}

Security C

Together AI Webhooks & Events

Overview

Together AI delivers webhook callbacks for asynchronous operations including fine-tuning jobs, batch inference, and model lifecycle events. Subscribe to events for fine-tune completion, job failures, model deprecation notices, and batch processing status to build automated ML pipelines without polling the jobs API.

Webhook Registration

const response = await fetch("https://api.together.xyz/v1/webhooks", {
  method: "POST",
  headers: {
    "Authorization": `Bearer ${process.env.TOGETHER_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://yourapp.com/webhooks/together",
    events: ["fine_tune.completed", "fine_tune.failed", "model.deprecated", "batch.done"],
    secret: process.env.TOGETHER_WEBHOOK_SECRET,
  }),
});

Signature Verification

import crypto from "crypto";
import { Request, Response, NextFunction } from "express";

function verifyTogetherSignature(req: Request, res: Response, next: NextFunction) {
  const signature = req.headers["x-together-signature"] as string;
  const expected = crypto.createHmac("sha256", process.env.TOGETHER_WEBHOOK_SECRET!)
    .update(req.body).digest("hex");
  if (!crypto.timingSafeEqual(Buffer.from(signature), Buffer.from(expected))) {
    return res.status(401).json({ error: "Invalid signature" });
  }
  next();
}

Event Handler

import express from "express";
const app = express();

app.post("/webhooks/together", express.raw({ type: "application/json" }), verifyTogetherSignature, (req, res) => {
  const event = JSON.parse(req.body.toString());
  res.status(200).json({ received: true });

  switch (event.type) {
    case "fine_tune.completed":
      deployModel(event.data.fine_tune_id, event.data.model_name); break;
    case "fine_tune.failed":
      alertTeam(event.data.fine_tune_id, event.data.error_message); break;
    case "model.deprecated":
      migratePipelines(event.data.model_id, event.data.replacement_model); break;
    case "batch.done":
      collectResults(event.data.batch_id, event.data.output_url); break;
  }
});

Event Types