go-ollama/generate_response.go

package ollama

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
)

type GenerateResponseRequest struct {
	Model       string                          `json:"model"`
	Prompt      string                          `json:"prompt,omitempty"`
	Suffix      string                          `json:"suffix,omitempty"`
	Images      []string                        `json:"images,omitempty"`
	Format      string                          `json:"format,omitempty"`
	System      string                          `json:"system,omitempty"`
	Stream      *bool                           `json:"stream,omitempty"`
	Think       *bool                           `json:"think,omitempty"`
	Raw         *bool                           `json:"raw,omitempty"`
	KeepAlive   string                          `json:"keep_alive,omitempty"`
	Options     *GenerateResponseRequestOptions `json:"options,omitempty"`
	Logprobs    *bool                           `json:"logprobs,omitempty"`
	TopLogprobs *int                            `json:"top_logprobs,omitempty"`
}

type GenerateResponseRequestOptions struct {
	Seed        *int     `json:"seed,omitempty"`
	Temperature *float32 `json:"temperature,omitempty"`
	TopK        *int     `json:"top_k,omitempty"`
	TopP        *float32 `json:"top_p,omitempty"`
	MinP        *float32 `json:"min_p,omitempty"`
	Stop        []string `json:"stop,omitempty"`
	NumCtx      *int     `json:"num_ctx,omitempty"`
	NumPredict  *int     `json:"num_predict,omitempty"`
}

type GenerateResponseResponse struct {
	Model              string `json:"model"`
	CreatedAt          string `json:"created_at"`
	Response           string `json:"response"`
	Thinking           string `json:"thinking"`
	Done               bool   `json:"done"`
	DoneReason         string `json:"done_reason"`
	TotalDuration      int    `json:"total_duration"`
	LoadDuration       int    `json:"load_duration"`
	PromptEvalCount    int    `json:"prompt_eval_count"`
	PromptEvalDuration int    `json:"prompt_eval_duration"`
	EvalCount          int    `json:"eval_count"`
	EvalDuration       int    `json:"eval_duration"`
	Logprobs           []struct {
		Token       string `json:"token"`
		Logprob     int    `json:"logprob"`
		Bytes       []int  `json:"bytes"`
		TopLogprobs []struct {
			Token   string `json:"token"`
			Logprob int    `json:"logprob"`
			Bytes   []int  `json:"bytes"`
		} `json:"top_logprobs"`
	} `json:"logprobs"`
}

type GenerateResponseResponseStream struct {
	Model              string `json:"model"`
	CreatedAt          string `json:"created_at"`
	Response           string `json:"response"`
	Thinking           string `json:"thinking"`
	Done               bool   `json:"done"`
	DoneReason         string `json:"done_reason"`
	TotalDuration      int    `json:"total_duration"`
	LoadDuration       int    `json:"load_duration"`
	PromptEvalCount    int    `json:"prompt_eval_count"`
	PromptEvalDuration int    `json:"prompt_eval_duration"`
	EvalCount          int    `json:"eval_count"`
	EvalDuration       int    `json:"eval_duration"`
}

func (o Ollama) GenerateResponse(reqBody GenerateResponseRequest) (GenerateResponseResponse, error) {
	reqBody.Stream = PtrOf(false)

	reqBodyBytes, err := json.Marshal(reqBody)
	if err != nil {
		return GenerateResponseResponse{}, err
	}

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/generate", o.baseUrl), bytes.NewReader(reqBodyBytes))
	if err != nil {
		return GenerateResponseResponse{}, err
	}

	for key, val := range o.customHeaders {
		req.Header.Set(key, val)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return GenerateResponseResponse{}, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return GenerateResponseResponse{}, errors.New("status code is not 200")
	}

	var respBody GenerateResponseResponse
	if err := json.NewDecoder(resp.Body).Decode(&respBody); err != nil {
		return GenerateResponseResponse{}, err
	}
	return respBody, nil
}

func (o Ollama) GenerateResponseStream(reqBody GenerateResponseRequest, onChunk func(chunk GenerateResponseResponseStream)) error {
	reqBody.Stream = PtrOf(true)

	reqBodyBytes, err := json.Marshal(reqBody)
	if err != nil {
		return err
	}

	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/generate", o.baseUrl), bytes.NewReader(reqBodyBytes))
	if err != nil {
		return err
	}

	for key, val := range o.customHeaders {
		req.Header.Set(key, val)
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return errors.New("status code is not 200")
	}

	scanner := bufio.NewScanner(resp.Body)

	for scanner.Scan() {
		line := bytes.TrimSpace(scanner.Bytes())

		var chunk GenerateResponseResponseStream
		if err := json.Unmarshal(line, &chunk); err != nil {
			return err
		}

		onChunk(chunk)
		if chunk.Done {
			return nil
		}
	}

	if err := scanner.Err(); err != nil {
		return err
	}

	return nil
}