DynamicMultiReader - a dynamic version of golangs standard io.MultiReader

dynamic-multi-reader.go

package dynamicmultireader

import "io"

var EOF = io.EOF

type dynamicMultiReader struct {
	currentReader    io.Reader
	newReaderFactory NewReaderFactory
	closeHandler     CloseHandler
}

func (mr *dynamicMultiReader) Read(p []byte) (n int, err error) {
	for mr.currentReader != nil {
		n, err = mr.currentReader.Read(p)
		if err == EOF {
			mr.currentReader = mr.newReaderFactory()
		}
		if n > 0 || err != EOF {
			if err == EOF && mr.currentReader != nil {
				// Don't return EOF yet. More readers remain.
				err = nil
			}
			return
		}
	}
	return 0, EOF
}

func (mr *dynamicMultiReader) Close() error {
	return mr.closeHandler()
}

type NewReaderFactory func() io.Reader
type CloseHandler func() error

// Works as standard io.MultiReader, but instead of accapting an array of readers
// this accepts readerFactory, which should return new readers, or nil, if no readers are remaining.
// This kind of DynamicMultiReader is needed when you get your source readers not at once, but sequentially.
func DynamicMultiReader(newReaderFactory NewReaderFactory, closeHandler CloseHandler) io.ReadCloser {
	initialReader := newReaderFactory()

	return &dynamicMultiReader{initialReader, newReaderFactory, closeHandler}
}

dynamic-multi-reader_test.go

package dynamicmultireader

import (
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"testing"
)

func TestDynamicMultiReader(t *testing.T) {
	data := `Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.`
	fmt.Printf("Len %d \n", len(data))

	originalReader := strings.NewReader(data)

	n := 0
	chunkSize := 444
	readSum := 0
	dmr := DynamicMultiReader(func() io.Reader {
		if readSum >= len(data) {
			return nil
		}

		fmt.Printf("Call to factory %d \n", n)
		n++
		readSum += chunkSize
		return io.LimitReader(originalReader, int64(chunkSize))
	}, func() error {
		return nil
	})

	out, err := ioutil.ReadAll(dmr)

	fmt.Printf("Err: %+v \n", err)

	outstr := string(out)

	fmt.Printf("Match: %t \n", data == outstr)

	fmt.Printf("Out: %+v \n", outstr)
}