GET a HTTP resource using multiple goroutines and mux the result into an output file

httpGetMux.go

package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"sync"
)

var (
	wg sync.WaitGroup
	mu sync.Mutex
)

func main() {
	var url, filename string
	var jobs int
	var err error
	var res *http.Response
	var length int
	var file *os.File

	flag.StringVar(&url, "url", "", "URL to fetch")
	flag.IntVar(&jobs, "jobs", 5, "number of jobs")
	flag.StringVar(&filename, "filename", "", "filename to write result to")
	flag.Parse()

	if url == "" || filename == "" {
		fmt.Println("url and filename must be specified")
		flag.PrintDefaults()
		os.Exit(1)
	}

	if jobs <= 0 {
		jobs = 1
	}

	file, err = os.OpenFile(filename+"_tmp", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	res, err = http.Head(url)
	if err != nil {
		fmt.Printf("error fetching HEAD: %s\n", err)
		os.Exit(1)
	}

	headers := res.Header
	length, err = strconv.Atoi(headers["Content-Length"][0])
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	chunkSize := length / jobs
	chunkSizeLast := length % jobs

	for i := 0; i < jobs; i++ {
		wg.Add(1)

		min := chunkSize * i
		max := chunkSize * (i + 1)

		if i == jobs-1 {
			max += chunkSizeLast
		}

		go Fetch(min, max, url, file)

	}
	wg.Wait()
}

func Fetch(min int, max int, url string, file *os.File) {
	client := &http.Client{}
	req, _ := http.NewRequest("GET", url, nil)
	range_header := "bytes=" + strconv.Itoa(min) + "-" + strconv.Itoa(max-1)
	req.Header.Add("Range", range_header)

	fmt.Printf("fetch range %d-%d\n", min, max)

	resp, err := client.Do(req)
	if err != nil {
		fmt.Printf("a: %s\n", err)
		os.Exit(1)
	}
	defer resp.Body.Close()

	reader := bufio.NewReader(resp.Body)

	read := 0
	for {
		var end bool
		line, err := reader.ReadBytes('\n')
		if err != nil {
			if err == io.EOF {
				end = true
			} else {
				fmt.Printf("b: %s\n", err)
				os.Exit(1)
			}
		}
		var count int
		mu.Lock()
		count, err = file.WriteAt(line, int64(min+read))
		mu.Unlock()
		read += len(line)
		if err != nil {
			fmt.Printf("c: %s\n", err)
			os.Exit(1)
		}

		if count != len(line) {
			fmt.Printf("write error: expected %d bytes, got %d bytes\n", len(line), count)
			os.Exit(1)
		}

		if end {
			break
		}
	}
	wg.Done()
}