Last active
August 29, 2015 14:00
-
-
Save caseyscarborough/3abd61d2780fa287291c to your computer and use it in GitHub Desktop.
A Java program I've written used to parse a test bank on Desire2Learn using Selenium and Jsoup, and a Groovy script that sorts the questions, and removes all the duplicates. Not meant to be efficient, just to get all questions from the test bank, an otherwise VERY tedious task.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import org.apache.commons.lang3.StringEscapeUtils; | |
import org.jsoup.Jsoup; | |
import org.jsoup.nodes.Document; | |
import org.jsoup.safety.Whitelist; | |
import org.jsoup.select.Elements; | |
import org.openqa.selenium.By; | |
import org.openqa.selenium.WebDriver; | |
import org.openqa.selenium.WebElement; | |
import org.openqa.selenium.firefox.FirefoxDriver; | |
import java.io.BufferedWriter; | |
import java.io.File; | |
import java.io.FileWriter; | |
import java.util.ArrayList; | |
import java.util.Collections; | |
public class D2LRipper { | |
public static void main(String[] args) { | |
WebDriver driver = new FirefoxDriver(); | |
driver.get("Desire2Learn Login URL"); | |
WebElement usernameField = driver.findElement(By.id("Username")); | |
WebElement passwordField = driver.findElement(By.id("Password")); | |
WebElement loginButton = driver.findElement(By.name("Login")); | |
usernameField.sendKeys("username"); | |
passwordField.sendKeys("password"); | |
loginButton.click(); | |
try { | |
for (int i = 0; i < 100; i++) { | |
driver.get("URL for 5 question self assessment"); | |
Thread.sleep(2000); | |
StringBuilder output = new StringBuilder(); | |
String content; | |
File file = new File("output.md"); | |
if (!file.exists()) { | |
file.createNewFile(); | |
} | |
FileWriter fw = new FileWriter(file.getAbsoluteFile(), true); | |
BufferedWriter bw = new BufferedWriter(fw); | |
content = driver.getPageSource(); | |
Document doc = Jsoup.parse(content, "Root URL"); | |
Elements elements = doc.select(".drt"); | |
for (int h = 0; h < elements.size(); h += elements.size() / 5) { | |
output.append(StringEscapeUtils.unescapeHtml4(Jsoup.clean(elements.get(h).text(), Whitelist.none()))); | |
output.append("\n"); | |
ArrayList<String> answers = new ArrayList<String>(); | |
for (int j = h + 1; j < h + 5; j++) { | |
answers.add(Jsoup.clean(elements.get(j).text(), Whitelist.none())); | |
} | |
Collections.sort(answers); | |
for (String s : answers) { | |
output.append("\n* "); | |
output.append(StringEscapeUtils.unescapeHtml4(s)); | |
} | |
output.append("\n\n"); | |
} | |
bw.write(output.toString()); | |
bw.close(); | |
} | |
} catch (Exception e) { | |
System.out.println("SHIT"); | |
e.printStackTrace(); | |
} | |
driver.quit(); | |
} | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
def file = new File("output.md") | |
def contents = file.readLines() | |
def questions = [] | |
print "Reading questions from file..." | |
for (int i = 0; i < contents.size(); i += 7) { | |
def question = new StringBuilder() | |
for (int j = 0; j < 7; j++) { | |
question << contents.get(j + i) + "\n"; | |
} | |
questions << question.toString() | |
} | |
println "done." | |
print "Sorting all questions..." | |
questions.sort { it.substring(0, 30) } | |
println "done." | |
print "Removing duplicate questions..." | |
questions.unique { a, b -> a <=> b } | |
println "done." | |
println "There are a total of ${questions.size()} unique questions." | |
new File("sorted.md").withWriter { out -> | |
questions.each() { q -> | |
out.write(q) | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment