playing with local AI, Ada vs Turing

main.go

kage main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"strings"
	"time"
)

const (
	serverURL = "http://127.0.0.1:8080/v1/chat/completions"
	model     = "gpt-3.5-turbo"

	turnCount    = 200
	debugTimings = true
)

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type ChatRequest struct {
	Model            string    `json:"model"`
	Messages         []Message `json:"messages"`
	Temperature      float64   `json:"temperature"`
	MaxTokens        int       `json:"max_tokens"`
	Stream           bool      `json:"stream"`
	CachePrompt      bool      `json:"cache_prompt"`
	FrequencyPenalty float64   `json:"frequency_penalty,omitempty"`
	PresencePenalty  float64   `json:"presence_penalty,omitempty"`
	TopP             float64   `json:"top_p,omitempty"`
	Stop             []string  `json:"stop,omitempty"`
}

type ChatResponse struct {
	Choices []struct {
		FinishReason string `json:"finish_reason"`
		Message      struct {
			Role             string `json:"role"`
			Content          string `json:"content"`
			ReasoningContent string `json:"reasoning_content"`
		} `json:"message"`
	} `json:"choices"`

	Usage struct {
		PromptTokens     int `json:"prompt_tokens"`
		CompletionTokens int `json:"completion_tokens"`
		TotalTokens      int `json:"total_tokens"`
	} `json:"usage"`

	Timings struct {
		CacheN             int     `json:"cache_n"`
		PromptN            int     `json:"prompt_n"`
		PromptMS           float64 `json:"prompt_ms"`
		PromptPerSecond    float64 `json:"prompt_per_second"`
		PredictedN         int     `json:"predicted_n"`
		PredictedMS        float64 `json:"predicted_ms"`
		PredictedPerSecond float64 `json:"predicted_per_second"`
	} `json:"timings"`

	Error *struct {
		Message string `json:"message"`
		Type    string `json:"type"`
	} `json:"error"`
}

type Turn struct {
	Speaker string
	Text    string
}

type CompletionResult struct {
	Content            string
	FinishReason       string
	ReasoningChars     int
	PromptTokens       int
	CompletionTokens   int
	TotalTokens        int
	CacheN             int
	PromptN            int
	PromptMS           float64
	PromptPerSecond    float64
	PredictedN         int
	PredictedMS        float64
	PredictedPerSecond float64
	MaxTokens          int
	Retried            bool
}

type EmptyLengthError struct {
	FinishReason   string
	ReasoningChars int
	MaxTokens      int
	Preview        string
}

func main() {
	client := &http.Client{
		Timeout: 180 * time.Second,
	}

	systemPrompt := `Scene:
Ada and Turing meet by chance at a quiet bar after a local AI meetup.
They are near the counter. The conversation should feel like two people talking, not like an essay.

Characters & Hidden Motives:
- Ada: Sharp, observant, dry. **Goal:** She suspects Turing has discovered a critical flaw in their company's core AI model, but he is hiding it. She is determined to corner him and make him confess the truth.
- Turing: Practical, calm, slightly ironic. **Goal:** He did find a catastrophic flaw, but he wants to fix it secretly before anyone panics. He is trying to deflect Ada's probing questions without outright lying, while keeping the secret safe.

Rules:
- Use Brazilian Portuguese.
- Keep it under 100 words.
- Say ONLY the spoken line of the requested character.
- Do NOT prepend the character's name to the line (e.g. avoid "Ada: " or "Turing: ").
- No bullet points.
- No stage directions or scene narration.
- No visible reasoning or explanations.
- React DIRECTLY to the previous line. Challenge the other person or ask direct questions.
- Maintain the topic of conversation. Do not be vague.
- Avoid starting sentences with filler words like "E", "Mas", or "Então".
- Be conversational, punchy, and direct.`

	messages := []Message{
		{
			Role:    "system",
			Content: systemPrompt,
		},
	}

	transcript := []Turn{}
	currentPrompt := "Turing is standing near the bar looking at the menu. Start with a natural first line. Reply as Ada."

	for turn := 1; turn <= turnCount; turn++ {
		speakerName := "Ada"
		nextSpeakerName := "Turing"
		if turn%2 == 0 {
			speakerName = "Turing"
			nextSpeakerName = "Ada"
		}

		messages = append(messages, Message{
			Role:    "user",
			Content: currentPrompt,
		})

		result, err := completeWithAdaptiveBudget(client, messages)
		if err != nil {
			fmt.Printf("error on turn %d (%s): %v\n", turn, speakerName, err)
			return
		}

		line := cleanLine(result.Content)
		if line == "" {
			fmt.Printf("error on turn %d (%s): cleaned line is empty\n", turn, speakerName)
			return
		}

		fmt.Printf("\n%s:\n%s\n", speakerName, line)

		messages = append(messages, Message{
			Role:    "assistant",
			Content: line,
		})

		transcript = append(transcript, Turn{
			Speaker: speakerName,
			Text:    line,
		})

		currentPrompt = "Generate " + nextSpeakerName + "'s reply to " + speakerName + "."
	}
}

func completeWithAdaptiveBudget(client *http.Client, messages []Message) (CompletionResult, error) {
	result, err := completeOnce(client, messages, 2048*4)
	if err == nil {
		return result, nil
	}

	if !isRetryableEmptyLengthError(err) {
		return CompletionResult{}, err
	}

	result, err = completeOnce(client, messages, 2048*8)
	if err != nil {
		return CompletionResult{}, err
	}

	result.Retried = true
	return result, nil
}

func completeOnce(client *http.Client, messages []Message, maxTokens int) (CompletionResult, error) {
	reqBody := ChatRequest{
		Model:            model,
		Messages:         messages,
		Temperature:      0.85,
		MaxTokens:        maxTokens,
		Stream:           false,
		CachePrompt:      true,
		FrequencyPenalty: 0.2,
		PresencePenalty:  0.0,
		TopP:             0.9,
		Stop: []string{
			"\nAda:",
			"\nTuring:",
			"\n\nAda:",
			"\n\nTuring:",
		},
	}

	rawBody, err := json.Marshal(reqBody)
	if err != nil {
		return CompletionResult{}, fmt.Errorf("marshal request: %w", err)
	}

	req, err := http.NewRequest(http.MethodPost, serverURL, bytes.NewReader(rawBody))
	if err != nil {
		return CompletionResult{}, fmt.Errorf("create request: %w", err)
	}

	req.Header.Set("Content-Type", "application/json")

	resp, err := client.Do(req)
	if err != nil {
		return CompletionResult{}, fmt.Errorf("call llama-server: %w", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	if err != nil {
		return CompletionResult{}, fmt.Errorf("read response: %w", err)
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		return CompletionResult{}, fmt.Errorf("llama-server returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
	}

	var chatResp ChatResponse
	if err := json.Unmarshal(body, &chatResp); err != nil {
		return CompletionResult{}, fmt.Errorf("decode response: %w\nraw response: %s", err, strings.TrimSpace(string(body)))
	}

	if chatResp.Error != nil {
		return CompletionResult{}, fmt.Errorf("server error: %s: %s", chatResp.Error.Type, chatResp.Error.Message)
	}

	if len(chatResp.Choices) == 0 {
		return CompletionResult{}, fmt.Errorf("empty choices\nraw response: %s", strings.TrimSpace(string(body)))
	}

	choice := chatResp.Choices[0]

	content := strings.TrimSpace(choice.Message.Content)
	reasoning := strings.TrimSpace(choice.Message.ReasoningContent)

	if content == "" {
		return CompletionResult{}, EmptyLengthError{
			FinishReason:   choice.FinishReason,
			ReasoningChars: len(reasoning),
			MaxTokens:      maxTokens,
			Preview:        truncate(reasoning, 160),
		}
	}

	if strings.EqualFold(content, "null") {
		return CompletionResult{}, errors.New("model returned null-like content")
	}

	return CompletionResult{
		Content:            content,
		FinishReason:       choice.FinishReason,
		ReasoningChars:     len(reasoning),
		PromptTokens:       chatResp.Usage.PromptTokens,
		CompletionTokens:   chatResp.Usage.CompletionTokens,
		TotalTokens:        chatResp.Usage.TotalTokens,
		CacheN:             chatResp.Timings.CacheN,
		PromptN:            chatResp.Timings.PromptN,
		PromptMS:           chatResp.Timings.PromptMS,
		PromptPerSecond:    chatResp.Timings.PromptPerSecond,
		PredictedN:         chatResp.Timings.PredictedN,
		PredictedMS:        chatResp.Timings.PredictedMS,
		PredictedPerSecond: chatResp.Timings.PredictedPerSecond,
		MaxTokens:          maxTokens,
	}, nil
}

func (e EmptyLengthError) Error() string {
	return fmt.Sprintf(
		"empty final output; finish_reason=%q; max_tokens=%d; reasoning_chars=%d; reasoning_preview=%q",
		e.FinishReason,
		e.MaxTokens,
		e.ReasoningChars,
		e.Preview,
	)
}

func isRetryableEmptyLengthError(err error) bool {
	var emptyErr EmptyLengthError
	if !errors.As(err, &emptyErr) {
		return false
	}

	return emptyErr.FinishReason == "length"
}

func cleanLine(s string) string {
	s = strings.TrimSpace(s)

	prefixes := []string{
		"Ada:",
		"Turing:",
		"ADA:",
		"TURING:",
		"**Ada:**",
		"**Turing:**",
		"\"Ada:\"",
		"\"Turing:\"",
	}

	for _, prefix := range prefixes {
		s = strings.TrimPrefix(s, prefix)
		s = strings.TrimSpace(s)
	}

	s = strings.Trim(s, `"`)

	lines := strings.SplitSeq(s, "\n")
	for line := range lines {
		line = strings.TrimSpace(line)
		if line != "" {
			return line
		}
	}

	return s
}

func truncate(s string, maxLen int) string {
	if len(s) <= maxLen {
		return s
	}

	return s[:maxLen] + "..."
}