Skip to content

Instantly share code, notes, and snippets.

@georgf

georgf/TelemetryArchiveValidation.js

Last active September 14, 2015 15:36
Download ZIP

Revisions

  1. georgf revised this gist Sep 14, 2015. 1 changed file with 21 additions and 5 deletions.
    26 changes: 21 additions & 5 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -372,15 +372,23 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    if (!v4Data.has(startDayUtc)) {
    v4Data.set(startDayUtc, []);
    }
    v4Data.get(startDayUtc).push({

    let entry = {
    startTime: startTimeUtc,
    sessionId: p.sessionId,
    totalTime: p.totalTime,
    aborted: p.reason == "aborted-session",
    searchCounts: p.searchCounts,
    sessionLength: p.sessionLength,
    subsessionLength: v4Extract.reduce((prev, curr) => prev + ((curr.sessionId == p.sessionId) ? curr.subsessionLength : 0), 0),
    });
    };
    // This fixes undefined sessionLength entries when the profile channel-switched to a build that
    // didn't have sessionLength yet (pre bug 1188416).
    if (entry.sessionLength === undefined) {
    entry.sessionLength = entry.subsessionLength;
    }

    v4Data.get(startDayUtc).push(entry);
    }

    // Filter out the v2 data points that have session starts.
    @@ -630,17 +638,20 @@ function renderV2V4Matchup(matchup) {
    text += "</table>";

    text += "<table>";
    text += `<tr><th>day/field</th><th>v2 totalTime</th><th>v4 totalTime</th><th>aborted</th><th>v4 session id</th></tr>`;
    text += `<tr><th>day/field</th><th>v2 totalTime</th><th>v4 totalTime</th><th>aborted</th><th>v4 session id</th><th>sessionLength</th><th>subsessionLength</th></tr>`;

    for (let [day, values] of matchup.sessions) {
    text += `<tr class='grey'><td>${day}</td><td></td><td></td><td></td><td></td></tr>`;
    text += `<tr class='grey'><td>${day}</td><td></td><td></td><td></td><td></td><td></td><td></td></tr>`;
    for (let v of values) {
    text += `<tr ${v.broken ? 'class="broken"' : ''}>` +
    `<td>${v.startTime > 0 ? new Date(v.startTime) : 'no v4 start time'}</td>` +
    `<td>${v.totalTimeV2 !== null ? v.totalTimeV2 : '-'}</td>` +
    `<td>${v.totalTimeV4 !== null ? v.totalTimeV4 : '-'}</td>` +
    `<td>${v.aborted}</td>` +
    `<td>${v.sessionId || '-'}</td></tr>`;
    `<td>${v.sessionId || '-'}</td>` +
    `<td>${v.sessionLength}</td>` +
    `<td>${v.subsessionLength}</td>` +
    `</tr>`;
    }
    }

    @@ -718,6 +729,8 @@ try {

    // Enable this for some additional data dumping.
    if (true) {
    v2v4Matchup.sessions = mapToObject(v2v4Matchup.sessions);

    text += "<h2>dumps</h2>" + "<pre>" +
    "Accumulated v2 data:\n" +
    JSON.stringify(v2Accumulated, null, 2) +
    @@ -732,6 +745,9 @@ try {
    "\n\n" +
    "Historic v2 data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2) +
    "\n\n" +
    "v2/v4 matchup data:\n" +
    JSON.stringify(v2v4Matchup, null, 2) +
    "</pre>";

    text += "<h2>Raw v2 data</h2>" +
  2. georgf revised this gist Sep 14, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -636,7 +636,7 @@ function renderV2V4Matchup(matchup) {
    text += `<tr class='grey'><td>${day}</td><td></td><td></td><td></td><td></td></tr>`;
    for (let v of values) {
    text += `<tr ${v.broken ? 'class="broken"' : ''}>` +
    `<td>${v.startTime > 0 ? new Date(v.startTime) : '-'}</td>` +
    `<td>${v.startTime > 0 ? new Date(v.startTime) : 'no v4 start time'}</td>` +
    `<td>${v.totalTimeV2 !== null ? v.totalTimeV2 : '-'}</td>` +
    `<td>${v.totalTimeV4 !== null ? v.totalTimeV4 : '-'}</td>` +
    `<td>${v.aborted}</td>` +
  3. @Dexterp37 Dexterp37 revised this gist Sep 8, 2015. 1 changed file with 2 additions and 0 deletions.
    2 changes: 2 additions & 0 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -111,6 +111,7 @@ function getV2Extract(rawV2Data) {
    const appinfo = data["org.mozilla.appInfo.appinfo"];
    if (appinfo) {
    extract.isDefaultBrowser = appinfo.isDefaultBrowser;
    dayHasData = true;
    }

    const previous = data["org.mozilla.appSessions.previous"];
    @@ -121,6 +122,7 @@ function getV2Extract(rawV2Data) {
    extract.abortedTotalTime = sum(extract.abortedTotalTimes);
    extract.cleanTotalTimes = (previous.cleanTotalTime || []);
    extract.cleanTotalTime = sum(extract.cleanTotalTimes);
    dayHasData = true;
    }

    if (dayHasData) {
  4. @Dexterp37 Dexterp37 revised this gist Sep 7, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -377,7 +377,7 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    aborted: p.reason == "aborted-session",
    searchCounts: p.searchCounts,
    sessionLength: p.sessionLength,
    subsessionLength: sessions.reduce((prev, curr) => prev + ((curr.sessionId == p.sessionId) ? curr.subsessionLength : 0), 0),
    subsessionLength: v4Extract.reduce((prev, curr) => prev + ((curr.sessionId == p.sessionId) ? curr.subsessionLength : 0), 0),
    });
    }

  5. georgf revised this gist Sep 3, 2015. 1 changed file with 27 additions and 1 deletion.
    28 changes: 27 additions & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -376,6 +376,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    totalTime: p.totalTime,
    aborted: p.reason == "aborted-session",
    searchCounts: p.searchCounts,
    sessionLength: p.sessionLength,
    subsessionLength: sessions.reduce((prev, curr) => prev + ((curr.sessionId == p.sessionId) ? curr.subsessionLength : 0), 0),
    });
    }

    @@ -429,6 +431,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    aborted: t.aborted,
    broken: !match,
    sessionId: match ? match.sessionId : null,
    sessionLength: match ? match.sessionLength : null,
    subsessionLength: match ? match.subsessionLength : null,
    });
    }
    }
    @@ -445,6 +449,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    aborted: p.aborted,
    broken: true,
    sessionId: p.sessionId,
    sessionLength: p.sessionLength,
    subsessionLength: p.subsessionLength,
    });
    }

    @@ -482,6 +488,22 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    values.reduce((p, c) => p + ((c.aborted && !c.broken) ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + ((c.aborted && !c.broken) ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    sessionLength: [
    0,
    values.reduce((p, c) => p + (c.sessionLength || 0), 0),
    ],
    matchedCleanSessionLength: [
    0,
    values.reduce((p, c) => p + ((!c.aborted && !c.broken) ? (c.sessionLength || 0) : 0), 0),
    ],
    subsessionLength: [
    0,
    values.reduce((p, c) => p + (c.subsessionLength || 0), 0),
    ],
    matchedCleanSubsessionLength: [
    0,
    values.reduce((p, c) => p + ((!c.aborted && !c.broken) ? (c.subsessionLength || 0) : 0), 0),
    ],
    };
    }

    @@ -583,17 +605,21 @@ function renderV2V4Matchup(matchup) {
    ["matchedCleanTotalTimes", "matchedCleanTotalTimes", 0.01],
    ["matchedTotalTimes", "matchedTotalTimes", 1.0],
    ["matchedAbortedTotalTimes", "matchedAbortedTotalTimes", 5.0],
    ["matchedTotalTimes", "matchedCleanSessionLength", 5.0],
    ["matchedTotalTimes", "matchedCleanSubsessionLength", 5.0],
    ["totalTimes", "totalTimes", 1.0],
    ["cleanTotalTimes", "cleanTotalTimes", 1.0],
    ["abortedTotalTimes", "abortedTotalTimes", 5.0],
    ["totalTimes", "sessionLength", 5.0],
    ["totalTimes", "subsessionLength", 5.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = matchup[fieldV2][0];
    const valV4 = matchup[fieldV4][1];
    const prop = valV4 / valV2;
    const broken = (delta(prop, 1.0) > tolerance);
    text += `<tr ${broken ? ' class="broken"' : ''}>` +
    `<td>${fieldV2} vs. ${fieldV4}</td>` +
    `<td>${fieldV2}${(fieldV2 != fieldV4) ? ' vs. ' + fieldV4 : ''}</td>` +
    `<td>${valV2}</td>` +
    `<td>${valV4}</td>` +
    `<td>${noInfinity(Math.round(prop * 1000) / 10)}%` +
  6. georgf revised this gist Sep 2, 2015. 1 changed file with 9 additions and 6 deletions.
    15 changes: 9 additions & 6 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -408,10 +408,13 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    let cleanTimes = [for (t of v2.cleanTotalTimes) {time: t, aborted: false}];
    let abortedTimes = [for (t of v2.abortedTotalTimes) {time: t, aborted: true}];
    for (let t of [...cleanTimes, ...abortedTimes]) {
    // We match up sessions via time with a delta D we'd expect for
    // totalTime matches for different collection times etc.
    const D = 5;
    let match = v4.find((p) => (p.aborted == t.aborted) &&
    (p.totalTime >= (t.time - D)) &&
    (p.totalTime <= (t.time + D)) &&
    (p.aborted ||
    (p.totalTime >= (t.time - D)) &&
    (p.totalTime <= (t.time + D))) &&
    !matchedSessionIds.has(p.sessionId));
    if (match) {
    matchedSessionIds.add(match.sessionId);
    @@ -577,11 +580,11 @@ function renderV2V4Matchup(matchup) {
    text += "<table>" +
    `<tr><th>what</th><th>v2</th><th>v4</th><th>v4 in % of v2</th></tr>`;
    fields = [
    ["matchedTotalTimes", "matchedTotalTimes", 0.01],
    ["matchedCleanTotalTimes", "matchedCleanTotalTimes", 0.01],
    ["matchedAbortedTotalTimes", "matchedAbortedTotalTimes", 0.01],
    ["totalTimes", "totalTimes", 0.05],
    ["cleanTotalTimes", "cleanTotalTimes", 0.05],
    ["matchedTotalTimes", "matchedTotalTimes", 1.0],
    ["matchedAbortedTotalTimes", "matchedAbortedTotalTimes", 5.0],
    ["totalTimes", "totalTimes", 1.0],
    ["cleanTotalTimes", "cleanTotalTimes", 1.0],
    ["abortedTotalTimes", "abortedTotalTimes", 5.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
  7. georgf revised this gist Sep 2, 2015. 1 changed file with 61 additions and 48 deletions.
    109 changes: 61 additions & 48 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -350,11 +350,17 @@ function validateV2V4BrowserDefault(v2, v4, cutoffTime) {
    }

    function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    // Lets preprocess the v2 data into a session-oriented format first.
    // Lets preprocess the v4 data into a session-oriented format first.
    let v4Data = new Map();
    const sessions = v4Extract.filter(p => p.isLastFragment);
    for (let p of sessions) {
    const startTimeUtc = (new Date(p.creationDate)).getTime() - (p.totalTime * 1000);
    let startTimeUtc = Date.now();
    if (p.reason != "gather-subsession-payload") {
    // We only insert the "current" session data on todays date, everything else
    // gets attributed to the session start date.
    startTimeUtc = (new Date(p.creationDate)).getTime() - (p.totalTime * 1000);

    }
    const startDate = new Date(startTimeUtc);
    const twoDig = (n) => ((n > 9) ? "" : "0") + n;
    const startDayUtc = `${startDate.getUTCFullYear()}-` +
    @@ -369,10 +375,11 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    sessionId: p.sessionId,
    totalTime: p.totalTime,
    aborted: p.reason == "aborted-session",
    searchCounts: p.searchCounts,
    });
    }

    // Filter out the v2 data that has session starts.
    // Filter out the v2 data points that have session starts.
    const v2Days = [for (v of v2Extract) if (v[1].totalTime > 0) v[0]];

    // Get the set of days we have either v2 or v4 sessions for.
    @@ -381,7 +388,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    days.reverse();

    // Build per-day session & match data.
    let data = new Map(); // day -> details
    const data = new Map(); // day -> details
    const matchedSessionIds = new Set();
    let missingInV2Count = 0;
    let missingInV4Count = 0;

    @@ -390,7 +398,6 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    continue;
    }

    let matchedSessionIds = new Set();
    let v2 = v2Extract.get(day);
    let v4 = v4Data.get(day) || [];
    let sessions = [];
    @@ -401,10 +408,10 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    let cleanTimes = [for (t of v2.cleanTotalTimes) {time: t, aborted: false}];
    let abortedTimes = [for (t of v2.abortedTotalTimes) {time: t, aborted: true}];
    for (let t of [...cleanTimes, ...abortedTimes]) {
    const D = 5 * 1000;
    const D = 5;
    let match = v4.find((p) => (p.aborted == t.aborted) &&
    (p.totalTime >= t.time - D) &&
    (p.totalTime <= t.time + D) &&
    (p.totalTime >= (t.time - D)) &&
    (p.totalTime <= (t.time + D)) &&
    !matchedSessionIds.has(p.sessionId));
    if (match) {
    matchedSessionIds.add(match.sessionId);
    @@ -414,8 +421,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {

    sessions.push({
    startTime: match ? match.startTime : 0,
    v2totalTime: t.time,
    v4totalTime: match ? match.totalTime : null,
    totalTimeV2: t.time,
    totalTimeV4: match ? match.totalTime : null,
    aborted: t.aborted,
    broken: !match,
    sessionId: match ? match.sessionId : null,
    @@ -430,8 +437,8 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    for (let p of unmatchedV4Sessions) {
    sessions.push({
    startTime: p.startTime,
    v2totalTime: null,
    v4totalTime: p.totalTime,
    totalTimeV2: null,
    totalTimeV4: p.totalTime,
    aborted: p.aborted,
    broken: true,
    sessionId: p.sessionId,
    @@ -449,20 +456,28 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    missingInV4Count: missingInV4Count,
    sessions: data,
    totalTimes: [
    values.reduce((p, c) => p + (c.v2totalTime || 0), 0),
    values.reduce((p, c) => p + (c.v4totalTime || 0), 0),
    values.reduce((p, c) => p + (c.totalTimeV2 || 0), 0),
    values.reduce((p, c) => p + (c.totalTimeV4 || 0), 0),
    ],
    matchedTotalTimes: [
    values.reduce((p, c) => p + (!c.broken ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.broken ? (c.v4totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.broken ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.broken ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    cleanTotalTimes: [
    values.reduce((p, c) => p + (!c.aborted ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.aborted ? (c.v4totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.aborted ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.aborted ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    matchedCleanTotalTimes: [
    values.reduce((p, c) => p + ((!c.aborted && !c.broken) ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + ((!c.aborted && !c.broken) ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    abortedTotalTimes: [
    values.reduce((p, c) => p + (c.aborted ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (c.aborted ? (c.v4totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (c.aborted ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + (c.aborted ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    matchedAbortedTotalTimes: [
    values.reduce((p, c) => p + ((c.aborted && !c.broken) ? (c.totalTimeV2 || 0) : 0), 0),
    values.reduce((p, c) => p + ((c.aborted && !c.broken) ? (c.totalTimeV4 || 0) : 0), 0),
    ],
    };
    }
    @@ -546,41 +561,40 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    text += "</tr>";
    }

    const fields = [
    ["totalTime", "totalTime", 0.05],
    ["totalTime", "subsessionLength", 0.05],
    ["totalTime", "sessionLength", 1.0],
    ["cleanTotalTime", "cleanTotalTime", 100.0],
    ["abortedTotalTime", "abortedTotalTime", 100.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = v2[fieldV2];
    const valV4 = v4[fieldV4];
    const prop = valV4 / valV2;
    const broken = (delta(prop, 1.0) > tolerance);
    text += `<tr ${broken ? ' class="broken"' : ''}>` +
    `<td>${fieldV2} vs. ${fieldV4}</td>` +
    `<td>${valV2}</td>` +
    `<td>${valV4}</td>` +
    `<td>${noInfinity(Math.round(prop * 1000) / 10)}%` +
    `</tr>`;
    }

    text += "</table>";
    return text;
    }

    function renderV2V4Matchup(matchup) {
    const delta = (a,b) => Math.abs(a - b);
    const noInfinity = (number) => (number == Infinity) ? "-" : number;

    let text = "<table>" +
    `<tr><td>v2 sessions not matched in v4</td><td>${matchup.missingInV4Count}</td></tr>` +
    `<tr><td>v4 sessions not matched in v2</td><td>${matchup.missingInV2Count}</td></tr>` +
    "</table>";

    text += "<table>" +
    `<tr><th>what</th><th>v2</th><th>v4</th></tr>`;
    const fields = ["totalTimes", "matchedTotalTimes", "cleanTotalTimes", "abortedTotalTimes"];
    for (let f of fields) {
    text += `<tr><td>${f}</td><td>${matchup[f][0]}</td><td>${matchup[f][1]}</td></tr>`;
    `<tr><th>what</th><th>v2</th><th>v4</th><th>v4 in % of v2</th></tr>`;
    fields = [
    ["matchedTotalTimes", "matchedTotalTimes", 0.01],
    ["matchedCleanTotalTimes", "matchedCleanTotalTimes", 0.01],
    ["matchedAbortedTotalTimes", "matchedAbortedTotalTimes", 0.01],
    ["totalTimes", "totalTimes", 0.05],
    ["cleanTotalTimes", "cleanTotalTimes", 0.05],
    ["abortedTotalTimes", "abortedTotalTimes", 5.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = matchup[fieldV2][0];
    const valV4 = matchup[fieldV4][1];
    const prop = valV4 / valV2;
    const broken = (delta(prop, 1.0) > tolerance);
    text += `<tr ${broken ? ' class="broken"' : ''}>` +
    `<td>${fieldV2} vs. ${fieldV4}</td>` +
    `<td>${valV2}</td>` +
    `<td>${valV4}</td>` +
    `<td>${noInfinity(Math.round(prop * 1000) / 10)}%` +
    `</tr>`;
    }
    text += "</table>";

    @@ -592,8 +606,8 @@ function renderV2V4Matchup(matchup) {
    for (let v of values) {
    text += `<tr ${v.broken ? 'class="broken"' : ''}>` +
    `<td>${v.startTime > 0 ? new Date(v.startTime) : '-'}</td>` +
    `<td>${v.v2totalTime !== null ? v.v2totalTime : '-'}</td>` +
    `<td>${v.v4totalTime !== null ? v.v4totalTime : '-'}</td>` +
    `<td>${v.totalTimeV2 !== null ? v.totalTimeV2 : '-'}</td>` +
    `<td>${v.totalTimeV4 !== null ? v.totalTimeV4 : '-'}</td>` +
    `<td>${v.aborted}</td>` +
    `<td>${v.sessionId || '-'}</td></tr>`;
    }
    @@ -662,9 +676,8 @@ try {
    "div { margin-bottom: 10px; }" +
    ".grey { background-color: #E3E3E3; }" +
    "</style>";
    text += "<h2 name='v2v4'>v2/v4 comparison</h2>";
    text += renderV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime, defaults);
    text += "<h2 name='v2v4matchup'>v2/v4 matchup</h2>";
    text += renderV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime, defaults);
    text += renderV2V4Matchup(v2v4Matchup);
    text += "<h2 name='v4'>v4 extract</h2>";
    v4Extract.reverse();
  8. georgf revised this gist Sep 1, 2015. 1 changed file with 26 additions and 1 deletion.
    27 changes: 26 additions & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -443,10 +443,27 @@ function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    data.set(day, sessions);
    }

    const values = [].concat(...[...data.values()]);
    return {
    missingInV2Count: missingInV2Count,
    missingInV4Count: missingInV4Count,
    sessions: data,
    totalTimes: [
    values.reduce((p, c) => p + (c.v2totalTime || 0), 0),
    values.reduce((p, c) => p + (c.v4totalTime || 0), 0),
    ],
    matchedTotalTimes: [
    values.reduce((p, c) => p + (!c.broken ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.broken ? (c.v4totalTime || 0) : 0), 0),
    ],
    cleanTotalTimes: [
    values.reduce((p, c) => p + (!c.aborted ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (!c.aborted ? (c.v4totalTime || 0) : 0), 0),
    ],
    abortedTotalTimes: [
    values.reduce((p, c) => p + (c.aborted ? (c.v2totalTime || 0) : 0), 0),
    values.reduce((p, c) => p + (c.aborted ? (c.v4totalTime || 0) : 0), 0),
    ],
    };
    }

    @@ -559,8 +576,16 @@ function renderV2V4Matchup(matchup) {
    `<tr><td>v4 sessions not matched in v2</td><td>${matchup.missingInV2Count}</td></tr>` +
    "</table>";

    text += "<table>" +
    `<tr><th>what</th><th>v2</th><th>v4</th></tr>`;
    const fields = ["totalTimes", "matchedTotalTimes", "cleanTotalTimes", "abortedTotalTimes"];
    for (let f of fields) {
    text += `<tr><td>${f}</td><td>${matchup[f][0]}</td><td>${matchup[f][1]}</td></tr>`;
    }
    text += "</table>";

    text += "<table>";
    text += `<tr><td>day/field</td><td>v2 totalTime</td><td>v4 totalTime</td><td>aborted</td><td>v4 session id</td></tr>`;
    text += `<tr><th>day/field</th><th>v2 totalTime</th><th>v4 totalTime</th><th>aborted</th><th>v4 session id</th></tr>`;

    for (let [day, values] of matchup.sessions) {
    text += `<tr class='grey'><td>${day}</td><td></td><td></td><td></td><td></td></tr>`;
  9. georgf revised this gist Sep 1, 2015. 1 changed file with 9 additions and 7 deletions.
    16 changes: 9 additions & 7 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -131,19 +131,21 @@ function getV2Extract(rawV2Data) {
    const twoDig = (n) => ((n > 9) ? "" : "0") + n;
    const dateStr = `${now.getUTCFullYear()}-${twoDig(now.getUTCMonth()+1)}-${twoDig(now.getUTCDate())}T00:00:00Z`;
    if (!dailyMap.has(dateStr)) {
    dailyMap.set(dateStr, {searchCounts: {}, totalTime: 0});
    dailyMap.set(dateStr, {
    searchCounts: {},
    totalTime: 0,
    cleanTotalTime: 0,
    cleanTotalTimes: [],
    abortedTotalTime: 0,
    abortedTotalTimes: [],
    });
    }
    const extract = dailyMap.get(dateStr);

    const current = payload.data.last["org.mozilla.appSessions.current"];
    extract.totalTime += current.totalTime;
    if (!extract.cleanTotalTimes) {
    extract.cleanTotalTimes = [];
    }
    extract.cleanTotalTime += current.totalTime;
    extract.cleanTotalTimes.push(current.totalTime);
    if (!extract.abortedTotalTimes) {
    extract.abortedTotalTimes = [];
    }

    return dailyMap;
    }
  10. georgf revised this gist Sep 1, 2015. 1 changed file with 154 additions and 3 deletions.
    157 changes: 154 additions & 3 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -137,6 +137,13 @@ function getV2Extract(rawV2Data) {

    const current = payload.data.last["org.mozilla.appSessions.current"];
    extract.totalTime += current.totalTime;
    if (!extract.cleanTotalTimes) {
    extract.cleanTotalTimes = [];
    }
    extract.cleanTotalTimes.push(current.totalTime);
    if (!extract.abortedTotalTimes) {
    extract.abortedTotalTimes = [];
    }

    return dailyMap;
    }
    @@ -234,11 +241,13 @@ function* getV4Extract() {

    let previous = null;
    for (let current of data) {
    const finalReasons = new Set(["shutdown", "aborted-session", "gather-subsession-payload"]);
    current.isFinalFragment = finalReasons.has(current.reason);

    // Check for consistency issues etc.
    if (previous) {
    const c = current;
    const p = previous;
    p.isFinalFragment = (p.reason == "shutdown" || p.reason == "aborted-session");
    p.isLastFragment = (p.sessionId != c.sessionId);

    c.channelSwitching = (c.channel != p.channel);
    @@ -268,6 +277,8 @@ function accumulateV4(extracts, cutoffTime) {
    let r = {
    searchCounts: {},
    totalTime: 0,
    cleanTotalTime: 0,
    abortedTotalTime: 0,
    subsessionLength: 0,
    sessionLength: 0,
    };
    @@ -284,6 +295,13 @@ function accumulateV4(extracts, cutoffTime) {
    r.subsessionLength += v.subsessionLength;
    if (v.isLastFragment) {
    r.totalTime += v.totalTime;

    if (v.reason == "aborted-session") {
    r.abortedTotalTime += v.totalTime;
    } else {
    r.cleanTotalTime += v.totalTime;
    }

    r.sessionLength += v.sessionLength || 0;
    }
    }
    @@ -328,6 +346,107 @@ function validateV2V4BrowserDefault(v2, v4, cutoffTime) {

    return sawBreakage;
    }

    function getV2V4Matchup(v2Extract, v4Extract, cutoffTime) {
    // Lets preprocess the v2 data into a session-oriented format first.
    let v4Data = new Map();
    const sessions = v4Extract.filter(p => p.isLastFragment);
    for (let p of sessions) {
    const startTimeUtc = (new Date(p.creationDate)).getTime() - (p.totalTime * 1000);
    const startDate = new Date(startTimeUtc);
    const twoDig = (n) => ((n > 9) ? "" : "0") + n;
    const startDayUtc = `${startDate.getUTCFullYear()}-` +
    `${twoDig(startDate.getUTCMonth() + 1)}-` +
    `${twoDig(startDate.getUTCDate())}` +
    `T00:00:00Z`;
    if (!v4Data.has(startDayUtc)) {
    v4Data.set(startDayUtc, []);
    }
    v4Data.get(startDayUtc).push({
    startTime: startTimeUtc,
    sessionId: p.sessionId,
    totalTime: p.totalTime,
    aborted: p.reason == "aborted-session",
    });
    }

    // Filter out the v2 data that has session starts.
    const v2Days = [for (v of v2Extract) if (v[1].totalTime > 0) v[0]];

    // Get the set of days we have either v2 or v4 sessions for.
    const days = [...(new Set([...v4Data.keys(), ...v2Days])).keys()];
    days.sort();
    days.reverse();

    // Build per-day session & match data.
    let data = new Map(); // day -> details
    let missingInV2Count = 0;
    let missingInV4Count = 0;

    for (let day of days) {
    if ((cutoffTime > 0) && (new Date(day).getTime() < cutoffTime)) {
    continue;
    }

    let matchedSessionIds = new Set();
    let v2 = v2Extract.get(day);
    let v4 = v4Data.get(day) || [];
    let sessions = [];

    // Add v2 sessions and try to match them with the v4 data.

    if (v2) {
    let cleanTimes = [for (t of v2.cleanTotalTimes) {time: t, aborted: false}];
    let abortedTimes = [for (t of v2.abortedTotalTimes) {time: t, aborted: true}];
    for (let t of [...cleanTimes, ...abortedTimes]) {
    const D = 5 * 1000;
    let match = v4.find((p) => (p.aborted == t.aborted) &&
    (p.totalTime >= t.time - D) &&
    (p.totalTime <= t.time + D) &&
    !matchedSessionIds.has(p.sessionId));
    if (match) {
    matchedSessionIds.add(match.sessionId);
    } else {
    ++missingInV4Count;
    }

    sessions.push({
    startTime: match ? match.startTime : 0,
    v2totalTime: t.time,
    v4totalTime: match ? match.totalTime : null,
    aborted: t.aborted,
    broken: !match,
    sessionId: match ? match.sessionId : null,
    });
    }
    }

    // Add unmatched v4 sessions.

    const unmatchedV4Sessions = v4.filter(p => !matchedSessionIds.has(p.sessionId));
    missingInV2Count += unmatchedV4Sessions.length;
    for (let p of unmatchedV4Sessions) {
    sessions.push({
    startTime: p.startTime,
    v2totalTime: null,
    v4totalTime: p.totalTime,
    aborted: p.aborted,
    broken: true,
    sessionId: p.sessionId,
    });
    }

    // Add the session matchups to the daily map.

    data.set(day, sessions);
    }

    return {
    missingInV2Count: missingInV2Count,
    missingInV4Count: missingInV4Count,
    sessions: data,
    };
    }

    function renderV4Extract(extract) {
    // Fields to print in the order we want them listed.
    @@ -412,8 +531,8 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    ["totalTime", "totalTime", 0.05],
    ["totalTime", "subsessionLength", 0.05],
    ["totalTime", "sessionLength", 1.0],
    ["cleanTotalTime", "totalTime", 100.0],
    ["abortedTotalTime", "totalTime", 100.0],
    ["cleanTotalTime", "cleanTotalTime", 100.0],
    ["abortedTotalTime", "abortedTotalTime", 100.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = v2[fieldV2];
    @@ -432,6 +551,32 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    return text;
    }

    function renderV2V4Matchup(matchup) {
    let text = "<table>" +
    `<tr><td>v2 sessions not matched in v4</td><td>${matchup.missingInV4Count}</td></tr>` +
    `<tr><td>v4 sessions not matched in v2</td><td>${matchup.missingInV2Count}</td></tr>` +
    "</table>";

    text += "<table>";
    text += `<tr><td>day/field</td><td>v2 totalTime</td><td>v4 totalTime</td><td>aborted</td><td>v4 session id</td></tr>`;

    for (let [day, values] of matchup.sessions) {
    text += `<tr class='grey'><td>${day}</td><td></td><td></td><td></td><td></td></tr>`;
    for (let v of values) {
    text += `<tr ${v.broken ? 'class="broken"' : ''}>` +
    `<td>${v.startTime > 0 ? new Date(v.startTime) : '-'}</td>` +
    `<td>${v.v2totalTime !== null ? v.v2totalTime : '-'}</td>` +
    `<td>${v.v4totalTime !== null ? v.v4totalTime : '-'}</td>` +
    `<td>${v.aborted}</td>` +
    `<td>${v.sessionId || '-'}</td></tr>`;
    }
    }

    text += "</table>";

    return text;
    }

    Task.spawn(function*() {
    try {
    let v4Extract = yield* getV4Extract();
    @@ -479,15 +624,21 @@ try {
    historicallyBroken: validateV2V4BrowserDefault(v2Extract, v4Extract, cutoffTime),
    };

    // v2/v4 session matchup.
    const v2v4Matchup = getV2V4Matchup(v2Extract, v4Extract, cutoffTime);

    // Show the v2 & v4 data and data comparisons in a new tab.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; margin-bottom: 10px; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "div { margin-bottom: 10px; }" +
    ".grey { background-color: #E3E3E3; }" +
    "</style>";
    text += "<h2 name='v2v4'>v2/v4 comparison</h2>";
    text += renderV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime, defaults);
    text += "<h2 name='v2v4matchup'>v2/v4 matchup</h2>";
    text += renderV2V4Matchup(v2v4Matchup);
    text += "<h2 name='v4'>v4 extract</h2>";
    v4Extract.reverse();
    text += renderV4Extract(v4Extract);
  11. georgf revised this gist Aug 31, 2015. 1 changed file with 2 additions and 1 deletion.
    3 changes: 2 additions & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -213,7 +213,8 @@ function* getV4Extract() {
    try {
    p = yield TelemetryArchive.promiseArchivedPingById(archived.id);
    } catch (e) {
    data.push({id: archived.id, timestampCreated: archived.timestampCreated, fileNotFound: true, isBroken: true})
    // data.push({id: archived.id, timestampCreated: archived.timestampCreated, fileNotFound: true, isBroken: true});
    continue;
    }

    // Skip all leading pings from build ids that are too old.
  12. georgf revised this gist Aug 26, 2015. 1 changed file with 3 additions and 1 deletion.
    4 changes: 3 additions & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -445,7 +445,6 @@ try {
    alert("No v2 data to compare yet.");
    return;
    }
    //showTextInNewTab(JSON.stringify(mapToObject(v2Extract), null, 2));

    // Build a v2/v4 comparison for the best matching historic data we can find.
    // If all v2 & v4 data is relatively recent, we use all available local data from both.
    @@ -514,6 +513,9 @@ try {

    text += "<h2>Raw v2 data</h2>" +
    "<pre>" + JSON.stringify(rawV2Data, null, 2) + "</pre>";

    text += "<h2>Raw v4 data</h2>" +
    "<pre>" + JSON.stringify(v4Extract, null, 2) + "</pre>";
    }

    showHtmlInNewTab(text);
  13. georgf revised this gist Aug 25, 2015. 1 changed file with 31 additions and 21 deletions.
    52 changes: 31 additions & 21 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -72,14 +72,19 @@ function* extractDailyV2Measurement(dailyMap, measurement, name) {
    }
    }

    function* getV2Extract() {
    function* getRawV2Data() {
    const reporter = Cc["@mozilla.org/datareporting/service;1"].getService().wrappedJSObject.healthReporter;
    yield reporter.onInit();
    const now = new Date();

    const payload = yield reporter.collectAndObtainJSONPayload(true);

    let dailyMap = new Map();
    return payload;
    }

    function getV2Extract(rawV2Data) {
    const now = new Date();
    const payload = rawV2Data;
    const dailyMap = new Map();

    for (let day of Object.keys(payload.data.days)) {
    const data = payload.data.days[day];
    let extract = {
    @@ -371,20 +376,23 @@ function renderV2Extract(data) {

    function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    const delta = (a,b) => Math.abs(a - b);
    const noInfinity = (number) => (number == Infinity) ? "-" : number;
    let text = "";

    // Some information.
    text += "<div>";
    text += `cutoff: ${new Date(cutoffTime)} ${(cutoffTime == 0) ? "<i>(all v2 & v4 data is recent)</i>" : ""}<br>`;
    text += "</div>";
    // Some basic information.
    text += "<table>";
    text += `<tr><td>cutoff date for inspections</td>` +
    `<td>${new Date(cutoffTime)} ${(cutoffTime == 0) ? "<i>(all v2 & v4 data is recent)</i>" : ""}</td></tr>`;

    text += "<table><tr><th>default</th><th>v2</th><th>v4</th></tr>";
    const currentBroken = (defaults.current.v2 != defaults.current.v4);
    text += `<tr ${currentBroken ? ' class="broken"' : ''}>` +
    `<td>current</td><td>${defaults.current.v2}</td><td>${defaults.current.v4}</td></tr>` +
    `<tr ${defaults.historicallyBroken ? ' class="broken"' : ''}>` +
    `<td>current</td><td>${defaults.historicallyBroken}</td><td>-</td></tr>`;
    text += "</table>";
    text += `<tr><td ${currentBroken ? ' class="broken"' : ''}>current browser defaults</td>` +
    `<td>v2: ${defaults.current.v2}, v4: ${defaults.current.v4}</td></tr>`;

    text += `<tr><td ${defaults.historicallyBroken ? ' class="broken"' : ''}>` +
    `History of browser defaults broken</td><td>${defaults.historicallyBroken}` +
    `</td></tr>`;

    text += "</div>";

    // Build comparison table.
    text += "<table>";
    @@ -395,7 +403,7 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    const broken = (delta(prop, 1.0) > 0.01);
    text += `<tr ${broken ? ' class="broken"' : ''}><td>${k}</td>`;
    text += `<td>${v[0]}</td><td>${v[1]}</td>`;
    text += `<td>${Math.round(prop * 1000) / 10}%`;
    text += `<td>${noInfinity(Math.round(prop * 1000) / 10)}%`;
    text += "</tr>";
    }

    @@ -415,7 +423,7 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    `<td>${fieldV2} vs. ${fieldV4}</td>` +
    `<td>${valV2}</td>` +
    `<td>${valV4}</td>` +
    `<td>${Math.round(prop * 1000) / 10}%` +
    `<td>${noInfinity(Math.round(prop * 1000) / 10)}%` +
    `</tr>`;
    }

    @@ -431,10 +439,8 @@ try {
    return;
    }

    // Show v4 extract in a new tab for consistency checking.


    let v2Extract = yield* getV2Extract();
    const rawV2Data = yield* getRawV2Data();
    let v2Extract = getV2Extract(rawV2Data);
    if (v2Extract.size == 0) {
    alert("No v2 data to compare yet.");
    return;
    @@ -478,6 +484,7 @@ try {
    "table { border-collapse: collapse; margin-bottom: 10px; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "div { margin-bottom: 10px; }" +
    "</style>";
    text += "<h2 name='v2v4'>v2/v4 comparison</h2>";
    text += renderV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime, defaults);
    @@ -488,7 +495,7 @@ try {
    text += renderV2Extract(v2Extract);

    // Enable this for some additional data dumping.
    if (false) {
    if (true) {
    text += "<h2>dumps</h2>" + "<pre>" +
    "Accumulated v2 data:\n" +
    JSON.stringify(v2Accumulated, null, 2) +
    @@ -504,6 +511,9 @@ try {
    "Historic v2 data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2) +
    "</pre>";

    text += "<h2>Raw v2 data</h2>" +
    "<pre>" + JSON.stringify(rawV2Data, null, 2) + "</pre>";
    }

    showHtmlInNewTab(text);
  14. georgf revised this gist Aug 25, 2015. 1 changed file with 59 additions and 6 deletions.
    65 changes: 59 additions & 6 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -53,6 +53,14 @@ function lastElement(array) {
    return array[array.length - 1];
    }

    function v2DefaultBrowserValueToV4(defaultValue) {
    switch (defaultValue) {
    case 1: return true;
    case 0: return false;
    default: return null;
    }
    }

    function* extractDailyV2Measurement(dailyMap, measurement, name) {
    let data = yield measurement.getValues();
    for (let [day,value] of [...Iterator(data.days)]) {
    @@ -276,7 +284,45 @@ function accumulateV4(extracts, cutoffTime) {

    return r;
    }


    function validateV2V4BrowserDefault(v2, v4, cutoffTime) {
    let sawBreakage = false;

    for (let [day,v2Data] of v2) {
    v2Data.brokenDefaultBrowser = false;

    const v2Time = (new Date(day)).getTime();
    if (v2Time < cutoffTime) {
    continue;
    }

    // Skip this entry if v2 didn't have any useful data.
    if (v2Data.isDefaultBrowser == null) {
    continue;
    }

    // Check for matching default entries in v4 from the same day +/- one day.
    // v2 & v4 don't match exactly, so we want to need to look
    // for matches a bit more loosely.
    // Also, v2 records this daily, v4 with every ping and hence potentially multiple
    // times a day. The best criteria here thus is "for each v2 entry, check that v4
    // also saw that value in a certain timeframe".

    v2Data.brokenDefaultBrowser = !v4.some((ping) => {
    const v4Time = (new Date(ping.creationDate)).getTime();
    if ((v4Time < (v2Time - MS_IN_A_DAY)) || (v4Time > (v2Time + MS_IN_A_DAY))) {
    return false;
    }

    return (ping.isDefaultBrowser === v2DefaultBrowserValueToV4(v2Data.isDefaultBrowser));
    });

    sawBreakage = sawBreakage || v2Data.brokenDefaultBrowser;
    }

    return sawBreakage;
    }

    function renderV4Extract(extract) {
    // Fields to print in the order we want them listed.
    const printFields = [
    @@ -308,12 +354,14 @@ function renderV2Extract(data) {
    let text = "";

    text += "<table>";
    text += `<tr><th>day</th><th>default</th><th>totalTime</th><th>cleanTotalTimes</th><th>abortedTotalTimes</th></tr>`;
    text += `<tr><th>day</th><th>default</th><th>totalTime</th><th>cleanTotalTimes</th><th>abortedTotalTimes</th><th>brokenDefaultBrowser</th></tr>`;
    for (let [day, value] of data) {
    text += `<tr>` +
    const broken = value.brokenDefaultBrowser;
    text += `<tr ${broken ? ' class="broken"' : ''}>` +
    `<td>${day}</td><td>${value.isDefaultBrowser}</td><td>${value.totalTime}</td>` +
    `<td>${value.cleanTotalTime} = ${JSON.stringify(value.cleanTotalTimes)}</td>` +
    `<td>${value.abortedTotalTime} = ${JSON.stringify(value.abortedTotalTimes)}</td>` +
    `<td>${value.brokenDefaultBrowser}</td>` +
    `</tr>`;
    }
    text += "</table>";
    @@ -331,7 +379,11 @@ function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    text += "</div>";

    text += "<table><tr><th>default</th><th>v2</th><th>v4</th></tr>";
    text += `<tr><td>current</td><td>${defaults.current.v2}</td><td>${defaults.current.v4}</td></tr>`;
    const currentBroken = (defaults.current.v2 != defaults.current.v4);
    text += `<tr ${currentBroken ? ' class="broken"' : ''}>` +
    `<td>current</td><td>${defaults.current.v2}</td><td>${defaults.current.v4}</td></tr>` +
    `<tr ${defaults.historicallyBroken ? ' class="broken"' : ''}>` +
    `<td>current</td><td>${defaults.historicallyBroken}</td><td>-</td></tr>`;
    text += "</table>";

    // Build comparison table.
    @@ -416,11 +468,12 @@ try {
    let defaults = {
    current: {
    v4: lastV4.isDefaultBrowser,
    v2: lastV2.isDefaultBrowser,
    v2: v2DefaultBrowserValueToV4(lastV2.isDefaultBrowser),
    },
    historicallyBroken: validateV2V4BrowserDefault(v2Extract, v4Extract, cutoffTime),
    };

    // Show the v2 & v4 data & comparisons in a new tab.
    // Show the v2 & v4 data and data comparisons in a new tab.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; margin-bottom: 10px; }" +
    "th, td { border: solid 1px; }" +
  15. georgf revised this gist Aug 24, 2015. 1 changed file with 3 additions and 3 deletions.
    6 changes: 3 additions & 3 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -5,10 +5,10 @@
    * - make it run as chrome: choose Environment -> Browser
    * - click "Run"
    *
    * After scanning the local archives this should open two new tabs which highlight
    * After scanning the local archives this should open a new tab which highlights
    * potential issues in color:
    * - one for v4 subsession consistency
    * - one for v2/v4 comparisons
    * - v2/v4 comparisons
    * - v4 subsession consistency
    */

    (function() {
  16. georgf revised this gist Aug 24, 2015. 1 changed file with 104 additions and 41 deletions.
    145 changes: 104 additions & 41 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -49,6 +49,10 @@ function mapToObject(m) {
    return o;
    }

    function lastElement(array) {
    return array[array.length - 1];
    }

    function* extractDailyV2Measurement(dailyMap, measurement, name) {
    let data = yield measurement.getValues();
    for (let [day,value] of [...Iterator(data.days)]) {
    @@ -71,8 +75,13 @@ function* getV2Extract() {
    for (let day of Object.keys(payload.data.days)) {
    const data = payload.data.days[day];
    let extract = {
    isDefaultBrowser: null,
    searchCounts: {},
    totalTime: 0,
    cleanTotalTime: 0,
    cleanTotalTimes: [],
    abortedTotalTime: 0,
    abortedTotalTimes: [],
    };
    let dayHasData = false;

    @@ -86,10 +95,19 @@ function* getV2Extract() {
    }
    }

    const appinfo = data["org.mozilla.appInfo.appinfo"];
    if (appinfo) {
    extract.isDefaultBrowser = appinfo.isDefaultBrowser;
    }

    const previous = data["org.mozilla.appSessions.previous"];
    if (previous) {
    const sum = (array) => array.reduce((a,b) => a+b, 0);
    extract.totalTime += sum(previous.cleanTotalTime || previous.abortedTotalTime);
    extract.totalTime = sum(previous.cleanTotalTime || previous.abortedTotalTime);
    extract.abortedTotalTimes = (previous.abortedTotalTime || []);
    extract.abortedTotalTime = sum(extract.abortedTotalTimes);
    extract.cleanTotalTimes = (previous.cleanTotalTime || []);
    extract.cleanTotalTime = sum(extract.cleanTotalTimes);
    }

    if (dayHasData) {
    @@ -114,6 +132,8 @@ function accumulateV2(dailyMap, cutoffTime) {
    let r = {
    searchCounts: {},
    totalTime: 0,
    cleanTotalTime: 0,
    abortedTotalTime: 0,
    };

    for (let [day,v] of dailyMap) {
    @@ -124,6 +144,8 @@ function accumulateV2(dailyMap, cutoffTime) {
    r.searchCounts[k] = (r.searchCounts[k] || 0) + v.searchCounts[k];
    }
    r.totalTime += v.totalTime;
    r.cleanTotalTime += v.cleanTotalTime;
    r.abortedTotalTime += v.abortedTotalTime;
    }

    return r;
    @@ -152,6 +174,7 @@ function extractV4DataFromPing(p, isFromOldBuild = false) {
    isFromOldBuild: isFromOldBuild,
    totalTime: simpleMeasurements.totalTime,
    searchCounts: {},
    isDefaultBrowser: p.environment.settings.isDefaultBrowser,
    };

    // Extract search counts.
    @@ -254,48 +277,62 @@ function accumulateV4(extracts, cutoffTime) {
    return r;
    }

    function showV4Extract(extract) {
    function renderV4Extract(extract) {
    // Fields to print in the order we want them listed.
    const printFields = [
    "pingId", "clientId", "reason", "channel", "buildId", "creationDate",
    "creationDate",
    "pingId", "clientId", "reason", "channel", "buildId",
    "isDefaultBrowser",
    "sessionLength", "subsessionLength", "totalTime",
    "sessionId", "previousSessionId", "subsessionId", "previousSubsessionId",
    "profileSubsessionCounter", "subsessionCounter",
    "sessionLength", "subsessionLength",
    "fileNotFound", "channelSwitching", "brokenSessionChain", "brokenSubsessionChain", "brokenProfileSubsessionCounter", "brokenSubsessionCounter",
    "fileNotFound", "channelSwitching",
    "brokenSessionChain", "brokenSubsessionChain", "brokenProfileSubsessionCounter", "brokenSubsessionCounter",
    ];

    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "</style>";
    let text = "";
    text += "<table>";
    text += "<tr>" + [for (f of printFields) `<th>${f}</th>`].join("") + "</tr>";
    text += "<tr>" + [for (f of printFields) `<th>${f}</th>`].join("") + "</tr>";
    for (let d of extract) {
    text += `<tr ${d.isBroken ? ' class="broken"' : ''}>`;
    text += [for (f of printFields) `<td title="${f}">${d[f] != undefined ? d[f] : "-"}</td>`].join("");
    text += "</tr>";
    }
    text += "</table>";

    showHtmlInNewTab(text);
    return text;
    }

    function showV2V4Comparison(countsMap, v2, v4, cutoffTime) {
    function renderV2Extract(data) {
    let text = "";

    text += "<table>";
    text += `<tr><th>day</th><th>default</th><th>totalTime</th><th>cleanTotalTimes</th><th>abortedTotalTimes</th></tr>`;
    for (let [day, value] of data) {
    text += `<tr>` +
    `<td>${day}</td><td>${value.isDefaultBrowser}</td><td>${value.totalTime}</td>` +
    `<td>${value.cleanTotalTime} = ${JSON.stringify(value.cleanTotalTimes)}</td>` +
    `<td>${value.abortedTotalTime} = ${JSON.stringify(value.abortedTotalTimes)}</td>` +
    `</tr>`;
    }
    text += "</table>";

    return text;
    }

    function renderV2V4Comparison(countsMap, v2, v4, cutoffTime, defaults) {
    const delta = (a,b) => Math.abs(a - b);
    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "div { margin-bottom: 20px; }" +
    "</style>";
    let text = "";

    // Some information.
    text += "<div>";
    text += `cutoff: ${new Date(cutoffTime)} ${(cutoffTime == 0) ? "<i>(all v2 & v4 data is recent)</i>" : ""}<br>`;
    text += "</div>"
    text += "</div>";

    text += "<table><tr><th>default</th><th>v2</th><th>v4</th></tr>";
    text += `<tr><td>current</td><td>${defaults.current.v2}</td><td>${defaults.current.v4}</td></tr>`;
    text += "</table>";

    // Build comparison table.
    text += "<table>";
    @@ -313,7 +350,9 @@ function showV2V4Comparison(countsMap, v2, v4, cutoffTime) {
    const fields = [
    ["totalTime", "totalTime", 0.05],
    ["totalTime", "subsessionLength", 0.05],
    ["totalTime", "sessionLength", 0.05]
    ["totalTime", "sessionLength", 1.0],
    ["cleanTotalTime", "totalTime", 100.0],
    ["abortedTotalTime", "totalTime", 100.0],
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = v2[fieldV2];
    @@ -329,8 +368,7 @@ function showV2V4Comparison(countsMap, v2, v4, cutoffTime) {
    }

    text += "</table>";

    showHtmlInNewTab(text);
    return text;
    }

    Task.spawn(function*() {
    @@ -342,7 +380,7 @@ try {
    }

    // Show v4 extract in a new tab for consistency checking.
    showV4Extract(v4Extract);


    let v2Extract = yield* getV2Extract();
    if (v2Extract.size == 0) {
    @@ -357,7 +395,7 @@ try {
    // we slice the data off at a roughly matching point.
    const haveOldV4Pings = v4Extract.some((p) => (parseInt(p.buildId, 10) < BUILDID_CUTOFF) ||
    (parseInt(p.version, 10) < 42));
    const oldestV2 = TelemetryUtils.truncateToDays(new Date([...v2Extract.keys()].pop())).getTime();
    const oldestV2 = TelemetryUtils.truncateToDays(new Date(lastElement([...v2Extract.keys()]))).getTime();
    const oldestV4 = TelemetryUtils.truncateToDays(new Date(v4Extract[0].creationDate)).getTime();
    let cutoffTime = Math.max(oldestV2, oldestV4);
    if (Math.abs(oldestV2 - oldestV4) <= (MS_IN_A_DAY)) {
    @@ -372,25 +410,50 @@ try {
    [(v2Accumulated.searchCounts[k] || 0), (v4Accumulated.searchCounts[k] || 0)]
    ]]);

    // Extract the current default browser.
    const lastV4 = lastElement(v4Extract);
    const lastV2 = v2Extract.values().next().value;
    let defaults = {
    current: {
    v4: lastV4.isDefaultBrowser,
    v2: lastV2.isDefaultBrowser,
    },
    };

    // Show the v2 & v4 data & comparisons in a new tab.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; margin-bottom: 10px; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "</style>";
    text += "<h2 name='v2v4'>v2/v4 comparison</h2>";
    text += renderV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime, defaults);
    text += "<h2 name='v4'>v4 extract</h2>";
    v4Extract.reverse();
    text += renderV4Extract(v4Extract);
    text += "<h2 name='v2'>v2 extract</h2>";
    text += renderV2Extract(v2Extract);

    // Enable this for some additional data dumping.
    if (false) {
    showTextInNewTab("Accumulated v2 data:\n" +
    JSON.stringify(v2Accumulated, null, 2) +
    "\n\n" +
    "Accumulated v4 data:\n" +
    JSON.stringify(v4Accumulated, null, 2) +
    "\n\n" +
    "Accumulation comparison for cutoffTime " + new Date(cutoffTime) + "\n" +
    "Oldest v2 date: " + new Date(oldestV2) + "\n" +
    "Oldest v4 date: " + new Date(oldestV4) + "\n" +
    JSON.stringify(mapToObject(counts), null, 2) +
    "\n\n" +
    "Historic v2 data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2));
    text += "<h2>dumps</h2>" + "<pre>" +
    "Accumulated v2 data:\n" +
    JSON.stringify(v2Accumulated, null, 2) +
    "\n\n" +
    "Accumulated v4 data:\n" +
    JSON.stringify(v4Accumulated, null, 2) +
    "\n\n" +
    "Accumulation comparison for cutoffTime " + new Date(cutoffTime) + "\n" +
    "Oldest v2 date: " + new Date(oldestV2) + "\n" +
    "Oldest v4 date: " + new Date(oldestV4) + "\n" +
    JSON.stringify(mapToObject(counts), null, 2) +
    "\n\n" +
    "Historic v2 data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2) +
    "</pre>";
    }

    // Show the v2/v4 comparison in a new tab.
    showV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime);
    showHtmlInNewTab(text);
    } catch (e) {
    alert(e + "\n" + e.stack);
    }
  17. georgf revised this gist Aug 21, 2015. 1 changed file with 103 additions and 20 deletions.
    123 changes: 103 additions & 20 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -17,6 +17,9 @@ Cu.import("resource://gre/modules/TelemetryArchive.jsm");
    Cu.import("resource://gre/modules/TelemetryController.jsm");
    Cu.import("resource://gre/modules/TelemetryUtils.jsm");

    const BUILDID_CUTOFF = 20150722000000;
    const MS_IN_A_DAY = 24 * 60 * 60 * 1000;

    function getMainWindow() {
    return window.QueryInterface(Ci.nsIInterfaceRequestor)
    .getInterface(Ci.nsIWebNavigation)
    @@ -66,31 +69,51 @@ function* getV2Extract() {

    let dailyMap = new Map();
    for (let day of Object.keys(payload.data.days)) {
    let data = payload.data.days[day];
    const data = payload.data.days[day];
    let extract = {
    searchCounts: {},
    totalTime: 0,
    };
    let dayHasData = false;

    let counts = data["org.mozilla.searches.counts"];
    const counts = data["org.mozilla.searches.counts"];
    if (counts) {
    for (let k of Object.keys(counts)) {
    if (k != "_v") {
    extract.searchCounts[k] = counts[k];
    dayHasData = true;
    }
    }
    }

    const previous = data["org.mozilla.appSessions.previous"];
    if (previous) {
    const sum = (array) => array.reduce((a,b) => a+b, 0);
    extract.totalTime += sum(previous.cleanTotalTime || previous.abortedTotalTime);
    }

    if (Object.keys(extract.searchCounts).length > 0) {
    if (dayHasData) {
    dailyMap.set(day + "T00:00:00Z", extract);
    }
    }

    const twoDig = (n) => ((n > 9) ? "" : "0") + n;
    const dateStr = `${now.getUTCFullYear()}-${twoDig(now.getUTCMonth()+1)}-${twoDig(now.getUTCDate())}T00:00:00Z`;
    if (!dailyMap.has(dateStr)) {
    dailyMap.set(dateStr, {searchCounts: {}, totalTime: 0});
    }
    const extract = dailyMap.get(dateStr);

    const current = payload.data.last["org.mozilla.appSessions.current"];
    extract.totalTime += current.totalTime;

    return dailyMap;
    }

    function accumulateV2(dailyMap, cutoffTime) {
    let r = {
    searchCounts: {},
    totalTime: 0,
    };

    for (let [day,v] of dailyMap) {
    @@ -100,6 +123,7 @@ function accumulateV2(dailyMap, cutoffTime) {
    for (let k of Object.keys(v.searchCounts)) {
    r.searchCounts[k] = (r.searchCounts[k] || 0) + v.searchCounts[k];
    }
    r.totalTime += v.totalTime;
    }

    return r;
    @@ -108,13 +132,15 @@ function accumulateV2(dailyMap, cutoffTime) {
    function extractV4DataFromPing(p, isFromOldBuild = false) {
    // Build up reduced and flat ping data to work on.
    const info = p.payload.info;
    const simpleMeasurements = p.payload.simpleMeasurements;
    let data = {
    pingId: p.id,
    clientId: p.clientId,
    reason: info.reason,
    creationDate: p.creationDate,
    channel: p.application.channel,
    buildId: p.application.buildId,
    version: p.application.version,
    sessionId: info.sessionId,
    subsessionId: info.subsessionId,
    previousSessionId: info.previousSessionId,
    @@ -124,6 +150,7 @@ function extractV4DataFromPing(p, isFromOldBuild = false) {
    sessionLength: info.sessionLength,
    subsessionLength: info.subsessionLength,
    isFromOldBuild: isFromOldBuild,
    totalTime: simpleMeasurements.totalTime,
    searchCounts: {},
    };

    @@ -154,7 +181,7 @@ function* getV4Extract() {
    }

    // Skip all leading pings from build ids that are too old.
    const isFromOldBuild = (parseInt(p.application.buildId, 10) < 20150722000000);
    const isFromOldBuild = (parseInt(p.application.buildId, 10) < BUILDID_CUTOFF);
    if (!foundNewerBuild && isFromOldBuild) {
    continue;
    }
    @@ -174,13 +201,14 @@ function* getV4Extract() {
    if (previous) {
    const c = current;
    const p = previous;
    const previousIsFinalFragment = (p.reason == "shutdown" || p.reason == "aborted-session");
    p.isFinalFragment = (p.reason == "shutdown" || p.reason == "aborted-session");
    p.isLastFragment = (p.sessionId != c.sessionId);

    c.channelSwitching = (c.channel != p.channel);
    c.brokenSessionChain = previousIsFinalFragment && (c.previousSessionId != p.sessionId);
    c.brokenSessionChain = p.isFinalFragment && (c.previousSessionId != p.sessionId);
    c.brokenSubsessionChain = (c.previousSubsessionId != p.subsessionId);
    c.brokenProfileSubsessionCounter = (c.profileSubsessionCounter != (p.profileSubsessionCounter + 1));
    c.brokenSubsessionCounter = (previousIsFinalFragment ?
    c.brokenSubsessionCounter = (p.isFinalFragment ?
    (c.subsessionCounter != 1) :
    (c.subsessionCounter != (p.subsessionCounter + 1)));
    c.isBroken = !c.isFromOldBuild && !p.isFromOldBuild &&
    @@ -194,21 +222,33 @@ function* getV4Extract() {
    previous = current;
    }

    data[data.length-1].isLastFragment = true;

    return data;
    }

    function accumulateV4(extracts, cutoffTime) {
    let r = {
    searchCounts: {},
    totalTime: 0,
    subsessionLength: 0,
    sessionLength: 0,
    };

    for (let v of extracts) {
    if ((new Date(v.creationDate)).getTime() < cutoffTime) {
    continue;
    }

    for (let k of Object.keys(v.searchCounts)) {
    r.searchCounts[k] = (r.searchCounts[k] || 0) + v.searchCounts[k];
    }

    r.subsessionLength += v.subsessionLength;
    if (v.isLastFragment) {
    r.totalTime += v.totalTime;
    r.sessionLength += v.sessionLength || 0;
    }
    }

    return r;
    @@ -242,21 +282,52 @@ function showV4Extract(extract) {
    showHtmlInNewTab(text);
    }

    function showV2V4Comparison(countsMap) {
    function showV2V4Comparison(countsMap, v2, v4, cutoffTime) {
    const delta = (a,b) => Math.abs(a - b);
    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "div { margin-bottom: 20px; }" +
    "</style>";

    // Some information.
    text += "<div>";
    text += `cutoff: ${new Date(cutoffTime)} ${(cutoffTime == 0) ? "<i>(all v2 & v4 data is recent)</i>" : ""}<br>`;
    text += "</div>"

    // Build comparison table.
    text += "<table>";
    text += "<tr><th>what</th><th>v2</th><th>v4</th></tr>";

    text += "<tr><th>what</th><th>v2</th><th>v4</th><th>v4 in % of v2</tr>";
    for (let [k, v] of countsMap) {
    const isBroken = (v[0] != v[1]);
    text += `<tr ${isBroken ? ' class="broken"' : ''}><td>${k}</td>`;
    text += [for (c of v) `<td>${c}</td>`].join("");
    const prop = v[1] / v[0];
    const broken = (delta(prop, 1.0) > 0.01);
    text += `<tr ${broken ? ' class="broken"' : ''}><td>${k}</td>`;
    text += `<td>${v[0]}</td><td>${v[1]}</td>`;
    text += `<td>${Math.round(prop * 1000) / 10}%`;
    text += "</tr>";
    }

    const fields = [
    ["totalTime", "totalTime", 0.05],
    ["totalTime", "subsessionLength", 0.05],
    ["totalTime", "sessionLength", 0.05]
    ];
    for (let [fieldV2, fieldV4, tolerance] of fields) {
    const valV2 = v2[fieldV2];
    const valV4 = v4[fieldV4];
    const prop = valV4 / valV2;
    const broken = (delta(prop, 1.0) > tolerance);
    text += `<tr ${broken ? ' class="broken"' : ''}>` +
    `<td>${fieldV2} vs. ${fieldV4}</td>` +
    `<td>${valV2}</td>` +
    `<td>${valV4}</td>` +
    `<td>${Math.round(prop * 1000) / 10}%` +
    `</tr>`;
    }

    text += "</table>";

    showHtmlInNewTab(text);
    @@ -269,6 +340,8 @@ try {
    alert("No v4 data to compare yet.");
    return;
    }

    // Show v4 extract in a new tab for consistency checking.
    showV4Extract(v4Extract);

    let v2Extract = yield* getV2Extract();
    @@ -277,15 +350,24 @@ try {
    return;
    }
    //showTextInNewTab(JSON.stringify(mapToObject(v2Extract), null, 2));


    // Build a v2/v4 comparison for the best matching historic data we can find.
    // If all v2 & v4 data is relatively recent, we use all available local data from both.
    // If the v4 data has older pings or v2s or v4s history starts more than 1 day apart,
    // we slice the data off at a roughly matching point.
    const haveOldV4Pings = v4Extract.some((p) => (parseInt(p.buildId, 10) < BUILDID_CUTOFF) ||
    (parseInt(p.version, 10) < 42));
    const oldestV2 = TelemetryUtils.truncateToDays(new Date([...v2Extract.keys()].pop())).getTime();
    const oldestV4 = TelemetryUtils.truncateToDays(new Date(v4Extract[0].creationDate)).getTime();
    const cutoffTime = Math.max(oldestV2, oldestV4);
    let cutoffTime = Math.max(oldestV2, oldestV4);
    if (Math.abs(oldestV2 - oldestV4) <= (MS_IN_A_DAY)) {
    cutoffTime = 0;
    }

    let v2Accumulated = accumulateV2(v2Extract, cutoffTime);
    let v4Accumulated = accumulateV4(v4Extract, cutoffTime);
    let searchKeys = new Set([...Object.keys(v2Accumulated.searchCounts), ...Object.keys(v4Accumulated.searchCounts)]);
    let counts = new Map([for (k of searchKeys) [
    const v2Accumulated = accumulateV2(v2Extract, cutoffTime);
    const v4Accumulated = accumulateV4(v4Extract, cutoffTime);
    const searchKeys = new Set([...Object.keys(v2Accumulated.searchCounts), ...Object.keys(v4Accumulated.searchCounts)]);
    const counts = new Map([for (k of searchKeys) [
    "search: " + k ,
    [(v2Accumulated.searchCounts[k] || 0), (v4Accumulated.searchCounts[k] || 0)]
    ]]);
    @@ -303,11 +385,12 @@ try {
    "Oldest v4 date: " + new Date(oldestV4) + "\n" +
    JSON.stringify(mapToObject(counts), null, 2) +
    "\n\n" +
    "Historic data:\n" +
    "Historic v2 data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2));
    }

    showV2V4Comparison(counts);
    // Show the v2/v4 comparison in a new tab.
    showV2V4Comparison(counts, v2Accumulated, v4Accumulated, cutoffTime);
    } catch (e) {
    alert(e + "\n" + e.stack);
    }
  18. georgf revised this gist Aug 14, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -15,7 +15,7 @@

    Cu.import("resource://gre/modules/TelemetryArchive.jsm");
    Cu.import("resource://gre/modules/TelemetryController.jsm");
    Cu.import("resource://gre/modules/TelemetryUtils.jsm");
    Cu.import("resource://gre/modules/TelemetryUtils.jsm");

    function getMainWindow() {
    return window.QueryInterface(Ci.nsIInterfaceRequestor)
  19. georgf revised this gist Aug 14, 2015. 1 changed file with 6 additions and 2 deletions.
    8 changes: 6 additions & 2 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -289,12 +289,16 @@ try {
    "search: " + k ,
    [(v2Accumulated.searchCounts[k] || 0), (v4Accumulated.searchCounts[k] || 0)]
    ]]);

    // Enable this for some additional data dumping.
    if (false) {
    showTextInNewTab(JSON.stringify(v2Accumulated, null, 2) +
    showTextInNewTab("Accumulated v2 data:\n" +
    JSON.stringify(v2Accumulated, null, 2) +
    "\n\n" +
    "Accumulated v4 data:\n" +
    JSON.stringify(v4Accumulated, null, 2) +
    "\n\n" +
    "Accumulation for cutoffTime " + new Date(cutoffTime) + "\n" +
    "Accumulation comparison for cutoffTime " + new Date(cutoffTime) + "\n" +
    "Oldest v2 date: " + new Date(oldestV2) + "\n" +
    "Oldest v4 date: " + new Date(oldestV4) + "\n" +
    JSON.stringify(mapToObject(counts), null, 2) +
  20. georgf revised this gist Aug 14, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -289,7 +289,7 @@ try {
    "search: " + k ,
    [(v2Accumulated.searchCounts[k] || 0), (v4Accumulated.searchCounts[k] || 0)]
    ]]);
    if (true) {
    if (false) {
    showTextInNewTab(JSON.stringify(v2Accumulated, null, 2) +
    "\n\n" +
    JSON.stringify(v4Accumulated, null, 2) +
  21. georgf revised this gist Aug 14, 2015. 1 changed file with 206 additions and 26 deletions.
    232 changes: 206 additions & 26 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -4,11 +4,18 @@
    * - open scratchpad: Tools -> Web Developer -> Scratchpad
    * - make it run as chrome: choose Environment -> Browser
    * - click "Run"
    *
    * After scanning the local archives this should open two new tabs which highlight
    * potential issues in color:
    * - one for v4 subsession consistency
    * - one for v2/v4 comparisons
    */

    (function() {

    Cu.import("resource://gre/modules/TelemetryArchive.jsm");
    Cu.import("resource://gre/modules/TelemetryController.jsm");
    Cu.import("resource://gre/modules/TelemetryUtils.jsm");

    function getMainWindow() {
    return window.QueryInterface(Ci.nsIInterfaceRequestor)
    @@ -30,15 +37,112 @@ function showHtmlInNewTab(str) {
    let tab = win.gBrowser.addTab("data:text/html," + encodeURIComponent(str));
    win.gBrowser.selectedTab = tab;
    }

    function mapToObject(m) {
    let o = {};
    for (let [k,v] of m) {
    o[k] = m.get(k);
    }
    return o;
    }

    function* extractDailyV2Measurement(dailyMap, measurement, name) {
    let data = yield measurement.getValues();
    for (let [day,value] of [...Iterator(data.days)]) {
    day = day.toISOString();
    if (!dailyMap.has(day)) {
    dailyMap.set(day, {});
    }
    dailyMap.get(day)[name] = mapToObject(value);
    }
    }

    function* getV2Extract() {
    const reporter = Cc["@mozilla.org/datareporting/service;1"].getService().wrappedJSObject.healthReporter;
    yield reporter.onInit();
    const now = new Date();

    const payload = yield reporter.collectAndObtainJSONPayload(true);

    let dailyMap = new Map();
    for (let day of Object.keys(payload.data.days)) {
    let data = payload.data.days[day];
    let extract = {
    searchCounts: {},
    };

    let counts = data["org.mozilla.searches.counts"];
    if (counts) {
    for (let k of Object.keys(counts)) {
    if (k != "_v") {
    extract.searchCounts[k] = counts[k];
    }
    }
    }

    Task.spawn(function*() {
    if (Object.keys(extract.searchCounts).length > 0) {
    dailyMap.set(day + "T00:00:00Z", extract);
    }
    }

    return dailyMap;
    }

    function accumulateV2(dailyMap, cutoffTime) {
    let r = {
    searchCounts: {},
    };

    for (let [day,v] of dailyMap) {
    if ((new Date(day)).getTime() < cutoffTime) {
    continue;
    }
    for (let k of Object.keys(v.searchCounts)) {
    r.searchCounts[k] = (r.searchCounts[k] || 0) + v.searchCounts[k];
    }
    }

    return r;
    }

    function extractV4DataFromPing(p, isFromOldBuild = false) {
    // Build up reduced and flat ping data to work on.
    const info = p.payload.info;
    let data = {
    pingId: p.id,
    clientId: p.clientId,
    reason: info.reason,
    creationDate: p.creationDate,
    channel: p.application.channel,
    buildId: p.application.buildId,
    sessionId: info.sessionId,
    subsessionId: info.subsessionId,
    previousSessionId: info.previousSessionId,
    previousSubsessionId: info.previousSubsessionId,
    profileSubsessionCounter: info.profileSubsessionCounter,
    subsessionCounter: info.subsessionCounter,
    sessionLength: info.sessionLength,
    subsessionLength: info.subsessionLength,
    isFromOldBuild: isFromOldBuild,
    searchCounts: {},
    };

    // Extract search counts.
    const h = p.payload.keyedHistograms["SEARCH_COUNTS"];
    for (let k of Object.keys(h)) {
    data.searchCounts[k] = h[k].sum;
    }

    return data;
    }

    function* getV4Extract() {
    // Retrieve a list of the archived main pings.
    let pings = yield TelemetryArchive.promiseArchivedPingList();
    pings = pings.filter(p => p.type == "main");

    // Load and process the pings.
    // Load and extract data from the archived pings.
    let data = [];
    let previous = null;
    let foundNewerBuild = false;

    for (let archived of pings) {
    @@ -56,26 +160,16 @@ Task.spawn(function*() {
    }
    foundNewerBuild = true;

    // Build up reduced and flat ping data to work on.
    const info = p.payload.info;
    let current = {
    pingId: p.id,
    clientId: p.clientId,
    reason: info.reason,
    creationDate: p.creationDate,
    channel: p.application.channel,
    buildId: p.application.buildId,
    sessionId: info.sessionId,
    subsessionId: info.subsessionId,
    previousSessionId: info.previousSessionId,
    previousSubsessionId: info.previousSubsessionId,
    profileSubsessionCounter: info.profileSubsessionCounter,
    subsessionCounter: info.subsessionCounter,
    sessionLength: info.sessionLength,
    subsessionLength: info.subsessionLength,
    isFromOldBuild: isFromOldBuild,
    };

    data.push(extractV4DataFromPing(p, isFromOldBuild));
    }

    // Push the current data on the list, otherwise we are missing
    // the measurements from the last subsession on.
    let current = TelemetryController.getCurrentPingData(true);
    data.push(extractV4DataFromPing(current));

    let previous = null;
    for (let current of data) {
    // Check for consistency issues etc.
    if (previous) {
    const c = current;
    @@ -97,10 +191,30 @@ Task.spawn(function*() {
    c.brokenSubsessionCounter);
    }

    data.push(current);
    previous = current;
    }

    return data;
    }

    function accumulateV4(extracts, cutoffTime) {
    let r = {
    searchCounts: {},
    };

    for (let v of extracts) {
    if ((new Date(v.creationDate)).getTime() < cutoffTime) {
    continue;
    }
    for (let k of Object.keys(v.searchCounts)) {
    r.searchCounts[k] = (r.searchCounts[k] || 0) + v.searchCounts[k];
    }
    }

    return r;
    }

    function showV4Extract(extract) {
    // Fields to print in the order we want them listed.
    const printFields = [
    "pingId", "clientId", "reason", "channel", "buildId", "creationDate",
    @@ -110,7 +224,6 @@ Task.spawn(function*() {
    "fileNotFound", "channelSwitching", "brokenSessionChain", "brokenSubsessionChain", "brokenProfileSubsessionCounter", "brokenSubsessionCounter",
    ];


    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    @@ -119,14 +232,81 @@ Task.spawn(function*() {
    "</style>";
    text += "<table>";
    text += "<tr>" + [for (f of printFields) `<th>${f}</th>`].join("") + "</tr>";
    for (let d of data) {
    for (let d of extract) {
    text += `<tr ${d.isBroken ? ' class="broken"' : ''}>`;
    text += [for (f of printFields) `<td title="${f}">${d[f] != undefined ? d[f] : "-"}</td>`].join("");
    text += "</tr>";
    }
    text += "</table>";

    showHtmlInNewTab(text);
    }

    function showV2V4Comparison(countsMap) {
    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "</style>";
    text += "<table>";
    text += "<tr><th>what</th><th>v2</th><th>v4</th></tr>";
    for (let [k, v] of countsMap) {
    const isBroken = (v[0] != v[1]);
    text += `<tr ${isBroken ? ' class="broken"' : ''}><td>${k}</td>`;
    text += [for (c of v) `<td>${c}</td>`].join("");
    text += "</tr>";
    }
    text += "</table>";

    showHtmlInNewTab(text);
    }

    Task.spawn(function*() {
    try {
    let v4Extract = yield* getV4Extract();
    if (v4Extract.length == 0) {
    alert("No v4 data to compare yet.");
    return;
    }
    showV4Extract(v4Extract);

    let v2Extract = yield* getV2Extract();
    if (v2Extract.size == 0) {
    alert("No v2 data to compare yet.");
    return;
    }
    //showTextInNewTab(JSON.stringify(mapToObject(v2Extract), null, 2));

    const oldestV2 = TelemetryUtils.truncateToDays(new Date([...v2Extract.keys()].pop())).getTime();
    const oldestV4 = TelemetryUtils.truncateToDays(new Date(v4Extract[0].creationDate)).getTime();
    const cutoffTime = Math.max(oldestV2, oldestV4);

    let v2Accumulated = accumulateV2(v2Extract, cutoffTime);
    let v4Accumulated = accumulateV4(v4Extract, cutoffTime);
    let searchKeys = new Set([...Object.keys(v2Accumulated.searchCounts), ...Object.keys(v4Accumulated.searchCounts)]);
    let counts = new Map([for (k of searchKeys) [
    "search: " + k ,
    [(v2Accumulated.searchCounts[k] || 0), (v4Accumulated.searchCounts[k] || 0)]
    ]]);
    if (true) {
    showTextInNewTab(JSON.stringify(v2Accumulated, null, 2) +
    "\n\n" +
    JSON.stringify(v4Accumulated, null, 2) +
    "\n\n" +
    "Accumulation for cutoffTime " + new Date(cutoffTime) + "\n" +
    "Oldest v2 date: " + new Date(oldestV2) + "\n" +
    "Oldest v4 date: " + new Date(oldestV4) + "\n" +
    JSON.stringify(mapToObject(counts), null, 2) +
    "\n\n" +
    "Historic data:\n" +
    JSON.stringify(mapToObject(v2Extract), null, 2));
    }

    showV2V4Comparison(counts);
    } catch (e) {
    alert(e + "\n" + e.stack);
    }
    });

    })();
  22. georgf revised this gist Aug 13, 2015. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -71,7 +71,7 @@ Task.spawn(function*() {
    previousSubsessionId: info.previousSubsessionId,
    profileSubsessionCounter: info.profileSubsessionCounter,
    subsessionCounter: info.subsessionCounter,
    sessionLength: info.subsessionLength,
    sessionLength: info.sessionLength,
    subsessionLength: info.subsessionLength,
    isFromOldBuild: isFromOldBuild,
    };
  23. georgf created this gist Aug 13, 2015.
    132 changes: 132 additions & 0 deletions TelemetryArchiveValidation.js
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,132 @@
    /*
    * This is to be used from a Firefox scratchpad:
    * - enable chrome devtools: in about:config set "devtools.chrome.enabled" to true
    * - open scratchpad: Tools -> Web Developer -> Scratchpad
    * - make it run as chrome: choose Environment -> Browser
    * - click "Run"
    */

    (function() {

    Cu.import("resource://gre/modules/TelemetryArchive.jsm");

    function getMainWindow() {
    return window.QueryInterface(Ci.nsIInterfaceRequestor)
    .getInterface(Ci.nsIWebNavigation)
    .QueryInterface(Ci.nsIDocShellTreeItem)
    .rootTreeItem
    .QueryInterface(Ci.nsIInterfaceRequestor)
    .getInterface(Ci.nsIDOMWindow);
    }

    function showTextInNewTab(str) {
    let win = getMainWindow();
    let tab = win.gBrowser.addTab("data:text/plain," + encodeURIComponent(str));
    win.gBrowser.selectedTab = tab;
    }

    function showHtmlInNewTab(str) {
    let win = getMainWindow();
    let tab = win.gBrowser.addTab("data:text/html," + encodeURIComponent(str));
    win.gBrowser.selectedTab = tab;
    }

    Task.spawn(function*() {
    // Retrieve a list of the archived main pings.
    let pings = yield TelemetryArchive.promiseArchivedPingList();
    pings = pings.filter(p => p.type == "main");

    // Load and process the pings.
    let data = [];
    let previous = null;
    let foundNewerBuild = false;

    for (let archived of pings) {
    let p;
    try {
    p = yield TelemetryArchive.promiseArchivedPingById(archived.id);
    } catch (e) {
    data.push({id: archived.id, timestampCreated: archived.timestampCreated, fileNotFound: true, isBroken: true})
    }

    // Skip all leading pings from build ids that are too old.
    const isFromOldBuild = (parseInt(p.application.buildId, 10) < 20150722000000);
    if (!foundNewerBuild && isFromOldBuild) {
    continue;
    }
    foundNewerBuild = true;

    // Build up reduced and flat ping data to work on.
    const info = p.payload.info;
    let current = {
    pingId: p.id,
    clientId: p.clientId,
    reason: info.reason,
    creationDate: p.creationDate,
    channel: p.application.channel,
    buildId: p.application.buildId,
    sessionId: info.sessionId,
    subsessionId: info.subsessionId,
    previousSessionId: info.previousSessionId,
    previousSubsessionId: info.previousSubsessionId,
    profileSubsessionCounter: info.profileSubsessionCounter,
    subsessionCounter: info.subsessionCounter,
    sessionLength: info.subsessionLength,
    subsessionLength: info.subsessionLength,
    isFromOldBuild: isFromOldBuild,
    };

    // Check for consistency issues etc.
    if (previous) {
    const c = current;
    const p = previous;
    const previousIsFinalFragment = (p.reason == "shutdown" || p.reason == "aborted-session");

    c.channelSwitching = (c.channel != p.channel);
    c.brokenSessionChain = previousIsFinalFragment && (c.previousSessionId != p.sessionId);
    c.brokenSubsessionChain = (c.previousSubsessionId != p.subsessionId);
    c.brokenProfileSubsessionCounter = (c.profileSubsessionCounter != (p.profileSubsessionCounter + 1));
    c.brokenSubsessionCounter = (previousIsFinalFragment ?
    (c.subsessionCounter != 1) :
    (c.subsessionCounter != (p.subsessionCounter + 1)));
    c.isBroken = !c.isFromOldBuild && !p.isFromOldBuild &&
    !c.channelSwitching &&
    (c.brokenSessionChain ||
    c.brokenSubsessionChain ||
    c.brokenProfileSubsessionCounter ||
    c.brokenSubsessionCounter);
    }

    data.push(current);
    previous = current;
    }

    // Fields to print in the order we want them listed.
    const printFields = [
    "pingId", "clientId", "reason", "channel", "buildId", "creationDate",
    "sessionId", "previousSessionId", "subsessionId", "previousSubsessionId",
    "profileSubsessionCounter", "subsessionCounter",
    "sessionLength", "subsessionLength",
    "fileNotFound", "channelSwitching", "brokenSessionChain", "brokenSubsessionChain", "brokenProfileSubsessionCounter", "brokenSubsessionCounter",
    ];


    // Print an html table from the data.
    let text = "<style type='text/css'>" +
    "table { border-collapse: collapse; }" +
    "th, td { border: solid 1px; }" +
    "tr.broken { background-color: yellow; }" +
    "</style>";
    text += "<table>";
    text += "<tr>" + [for (f of printFields) `<th>${f}</th>`].join("") + "</tr>";
    for (let d of data) {
    text += `<tr ${d.isBroken ? ' class="broken"' : ''}>`;
    text += [for (f of printFields) `<td title="${f}">${d[f] != undefined ? d[f] : "-"}</td>`].join("");
    text += "</tr>";
    }
    text += "</table>";

    showHtmlInNewTab(text);
    });

    })();