mva-scraper.go

// MIT License
// 
// Copyright (c) 2019 xarantolus
// 
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
// 
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
// 
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"
)

const (
	// API url used by the search on the page
	apiURLFormat = "https://api-mlxprod.microsoft.com/sdk/search/v1.0/5/courses?$skip=%d&$top=%d"

	// The url format used by the course - %s is the language code and %d the course id. The `test-` can be replaced by anything else that contains a '-', but youtube-dl needs it to be in there
	courseURLFormat = "https://mva.microsoft.com/%s/training-courses/test-%d"

	// jsonBody describes the body that needs to be sent to the url above to get all courses. Note that all `SelectTerm` entries are set to null to select everything
	jsonBody = `{
		"SelectCriteria": [
			{
				"SelectOnField": "Audiences",
				"SelectTerm": null,
				"SelectMatchOption": 0
			},
			{
				"SelectOnField": "LCID",
				"SelectTerm": null,
				"SelectMatchOption": 0
			},
			{
				"SelectOnField": "Topics",
				"SelectTerm": null,
				"SelectMatchOption": 0
			}
		],
		"DisplayFields": [],
		"SortOptions": [
			{
				"SortOnField": "PublishedTime",
				"SortOrder": 1
			}
		],
		"SearchKeyword": "",
		"UILangaugeCode": null,
		"UserLanguageCode": null
	}`
)

var (
	client = http.Client{Timeout: 5 * time.Minute}
)

func main() {
	results := make(chan Course)
	go func() {
		err := requestCourses(results, 0)
		if err != nil {
			panic(err)
		}
		close(results)
	}()

	for course := range results {
		// EDIT HERE for any information you want to get

		fmt.Printf("%s\n", course.Link())
	}
}

// requestCourses requests 100 courses and puts them into the `results` channel
func requestCourses(results chan Course, skip int) (err error) {
	url := fmt.Sprintf(apiURLFormat, skip, 100)

	req, err := client.Post(url, "application/json", strings.NewReader(jsonBody))
	if err != nil {
		return
	}
	defer req.Body.Close()

	if req.StatusCode != 200 {
		return fmt.Errorf("Expected status code 200 but got %d when requesting \"%s\"", req.StatusCode, url)
	}

	dec := json.NewDecoder(req.Body)

	// Go to opening token
	_, err = dec.Token()
	if err != nil {
		return
	}

	// Go to `results` token
	_, err = dec.Token()
	if err != nil {
		return
	}

	// Go to first `results` item
	_, err = dec.Token()
	if err != nil {
		return
	}

	// Decode JSON
	for dec.More() {
		// Start decoding
		course := new(Course)

		err = dec.Decode(course)
		if err != nil {
			return
		}

		// Return this to channel
		results <- *course
		skip++
	}

	_, err = dec.Token()
	if err != nil {
		return
	}

	_, err = dec.Token()
	if err != nil {
		return
	}

	resp := new(int)
	// The `totalResultCount` field comes now
	err = dec.Decode(resp)
	if err != nil {
		return
	}

	// If we have all results, stop this
	if *resp == skip {
		return
	}

	// Continue with the next page of results
	return requestCourses(results, skip)
}

// From https://mholt.github.io/json-to-go/
type Course struct {
	ID                      int         `json:"id"`
	CourseNumber            string      `json:"courseNumber"`
	CourseLevel             string      `json:"courseLevel"`
	LanguageCode            string      `json:"languageCode"`
	LanguageID              string      `json:"languageId"`
	CourseName              string      `json:"courseName"`
	CourseShortDescription  string      `json:"courseShortDescription"`
	CourseDuration          interface{} `json:"courseDuration"`
	CourseImage             string      `json:"courseImage"`
	PublishedTime           string      `json:"publishedTime"`
	RetirementTime          interface{} `json:"retirementTime"`
	CourseStatus            string      `json:"courseStatus"`
	ProductPackageVersionID int         `json:"productPackageVersionId"`
	TotalModules            int         `json:"totalModules"`
	Version                 string      `json:"version"`
	Tags                    string      `json:"tags"`
	Points                  int         `json:"points"`
	LiveEvent               struct {
		LiveEventID     interface{}   `json:"liveEventId"`
		RegistrationURL interface{}   `json:"registrationURL"`
		EventURL        interface{}   `json:"eventURL"`
		EventSchedule   []interface{} `json:"eventSchedule"`
	} `json:"liveEvent"`
	ProductPackageVersionTypeName interface{} `json:"productPackageVersionTypeName"`
	LastUpdated                   string      `json:"lastUpdated"`
	IsLiveEventExisting           bool        `json:"isLiveEventExisting"`
	LiveEventStartDate            interface{} `json:"liveEventStartDate"`
	CourseFormats                 interface{} `json:"courseFormats"`
	IsVersionModified             bool        `json:"isVersionModified"`
	Technologies                  []string    `json:"technologies"`
	TechnologiesCategory          []string    `json:"technologiesCategory"`
	Audiences                     []string    `json:"audiences"`
	Topics                        []string    `json:"topics"`
	AuthorInfo                    []struct {
		AuthorID        int         `json:"authorID"`
		CreateDatetime  string      `json:"createDatetime"`
		ChangeDatetime  string      `json:"changeDatetime"`
		CreatedByUserID int         `json:"createdByUserID"`
		UpdatedByUserID int         `json:"updatedByUserID"`
		IsActive        interface{} `json:"isActive"`
		TwitterHandle   interface{} `json:"twitterHandle"`
		PersonalBio     interface{} `json:"personalBio"`
		ImageURL        interface{} `json:"imageUrl"`
		PersonalWebsite interface{} `json:"personalWebsite"`
		EmailAddress    interface{} `json:"emailAddress"`
		DisplayName     string      `json:"displayName"`
		FirstName       string      `json:"firstName"`
		LastName        string      `json:"lastName"`
		JobTitle        string      `json:"jobTitle"`
		Company         string      `json:"company"`
	} `json:"authorInfo"`
	ReplacementCourseDetails struct {
		CourseID                int         `json:"courseId"`
		CourseNumber            interface{} `json:"courseNumber"`
		LanguageCode            interface{} `json:"languageCode"`
		LanguageID              interface{} `json:"languageId"`
		CourseName              interface{} `json:"courseName"`
		Version                 interface{} `json:"version"`
		ProductPackageVersionID int         `json:"productPackageVersionID"`
	} `json:"replacementCourseDetails"`
	CourseRatings []struct {
		RatingCount        int     `json:"ratingCount"`
		AverageRatingValue float64 `json:"averageRatingValue"`
		ChannelID          int     `json:"channelId"`
	} `json:"courseRatings"`
}

func (c *Course) Link() string {
	return fmt.Sprintf(courseURLFormat, c.LanguageCode, c.ID)
}

type Response struct {
	Results          []Course `json:"results"`
	TotalResultCount int      `json:"totalResultCount"`
	NarrowBySections []struct {
		Header          string      `json:"header"`
		LocalizedHeader interface{} `json:"localizedHeader"`
		NarrowByInfos   []struct {
			Name          string `json:"name"`
			LocalizedName string `json:"localizedName"`
			Count         int    `json:"count"`
		} `json:"narrowByInfos"`
	} `json:"narrowBySections"`
	SearchIndexUpdatedDateTime string `json:"searchIndexUpdatedDateTime"`
}