Evetech

main.go

package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/signal"
	"strconv"
	"time"

	"golang.org/x/sync/errgroup"
	"golang.org/x/sys/unix"
)

func main() {
	defer measure(time.Now(), "evetech-1")

	if err := run(context.Background()); err != nil {
		fmt.Fprintf(os.Stderr, "ERR: %v\n", err)
	}
}

func run(ctx context.Context) error {
	ctx, cancel := signal.NotifyContext(ctx, unix.SIGINT, unix.SIGTERM, unix.SIGKILL)
	defer cancel()

	f, err := os.Create("evetech-1.csv")
	if err != nil {
		return err
	}
	defer f.Close()

	w := csv.NewWriter(f)
	defer w.Flush()

	header := []string{
		"duration",
		"is_buy_order",
		"issued",
		"location_id",
		"min_volume",
		"order_id",
		"price",
		"range",
		"system_id",
		"type_id",
		"volume_remain",
		"volume_total",
	}
	if err := w.Write(header); err != nil {
		return err
	}

	// get all ids
	// https://www.loginradius.com/blog/engineering/tune-the-go-http-client-for-high-performance
	t := http.DefaultTransport.(*http.Transport).Clone()
	c := &http.Client{
		Transport: t,
	}

	ids, err := ids(ctx, c)
	if err != nil {
		return err
	}

	g, ctx := errgroup.WithContext(ctx)
	g.SetLimit(1)

	records := make(chan []string, 100)

	for _, id := range ids {
		g.Go(func() error {
			orders, err := orders(ctx, c, id, 1)
			if err != nil {
				return err
			}
			for _, order := range orders {
				select {
				case records <- []string{
					strconv.Itoa(order.Duration),
					strconv.FormatBool(order.IsBuyOrder),
					order.Issued,
					strconv.FormatInt(order.LocationID, 10),
					strconv.Itoa(order.MinVolume),
					strconv.FormatInt(order.OrderID, 10),
					strconv.FormatFloat(order.Price, 'f', -1, 64),
					order.Range,
					strconv.FormatInt(order.SystemID, 10),
					strconv.Itoa(order.TypeID),
					strconv.Itoa(order.VolumeRemain),
					strconv.Itoa(order.VolumeTotal),
				}:
				case <-ctx.Done():
					return ctx.Err()
				}
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(records)
	}()
	for record := range records {
		if err := w.Write(record); err != nil {
			return err
		}
	}
	w.Flush()
	if err := g.Wait(); err != nil {
		return err
	}
	return nil
}

func ids(ctx context.Context, c *http.Client) ([]int, error) {
	url := "https://esi.evetech.net/v1/universe/regions"

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer func() { debug("ids: %v = resp.Body.Close()", resp.Body.Close()) }()

	var nn []int
	err = json.NewDecoder(resp.Body).Decode(&nn)
	if err != nil {
		return nil, err
	}
	return nn, nil
}

func pages(ctx context.Context, c *http.Client, id int) (int, error) {
	url := fmt.Sprintf("https://esi.evetech.net/v1/markets/%d/orders", id)
	req, err := http.NewRequestWithContext(ctx, http.MethodHead, url, nil) // context
	if err != nil {
		return 0, err
	}
	debug("pages:  http.NewRequestWithContext(method=head, url=%s)", url)
	resp, err := c.Do(req)
	if err != nil {
		return 0, err
	}
	defer func() { debug("pages: %v = resp.Body.Close()", resp.Body.Close()) }()

	pages, err := strconv.Atoi(resp.Header.Get("x-pages"))
	if err != nil {
		return 0, err
	}
	return pages, nil
}

type Order struct {
	Duration     int     `json:"duration"`
	IsBuyOrder   bool    `json:"is_buy_order"`
	Issued       string  `json:"issued"`
	LocationID   int64   `json:"location_id"`
	MinVolume    int     `json:"min_volume"`
	OrderID      int64   `json:"order_id"`
	Price        float64 `json:"price"`
	Range        string  `json:"range"`
	SystemID     int64   `json:"system_id"`
	TypeID       int     `json:"type_id"`
	VolumeRemain int     `json:"volume_remain"`
	VolumeTotal  int     `json:"volume_total"`
}

func orders(ctx context.Context, c *http.Client, id, page int) ([]Order, error) {
	url := fmt.Sprintf("https://esi.evetech.net/v1/markets/%d/orders?page=%d", id, page)

	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, err
	}
	debug("orders:  http.NewRequestWithContext(method=get, url=%s)", url)
	resp, err := c.Do(req)
	if err != nil {
		return nil, err
	}
	defer func() { debug("orders: %v = resp.Body.Close()", resp.Body.Close()) }()

	var nn []Order
	err = json.NewDecoder(resp.Body).Decode(&nn)
	if err != nil {
		return nil, err
	}

	return nn, nil
}

func measure(start time.Time, name string) {
	elapsed := time.Since(start)
	debug("%s took %s", name, elapsed)
}

var debug = log.Printf