Demo of exporting CSV in polis export format from API data, via red-dwarf python library.

polis-csv-export-demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/patcon/c768fb0d009fa5533beaccfe54e5c6b9/polis-csv-export-demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kEyVHx6y7zpu",
        "outputId": "9eb1e6a1-816d-4ddf-ac76-07b1cfeae4ce"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m647.5/647.5 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m161.7/161.7 kB\u001b[0m \u001b[31m14.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m61.4/61.4 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m70.0/70.0 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.0/116.0 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for red-dwarf (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for annoy (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ],
      "source": [
        "%pip install --quiet red-dwarf[all]@git+https://github.com/polis-community/red-dwarf.git@main"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "NkGdoHCy8RdA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b443773-7980-4a75-b9db-cf515eaececa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Loading data from https://pol.is/report/r2dfw8eambusb8buvecjt\n",
            "{\n",
            "  \"participant_id\": 0,\n",
            "  \"statement_id\": 0,\n",
            "  \"vote\": 1,\n",
            "  \"weight_x_32767\": 0,\n",
            "  \"modified\": 1544544810393.0,\n",
            "  \"conversation_id\": \"6jrufhr6dp\",\n",
            "  \"datetime\": null\n",
            "}\n",
            "{\n",
            "  \"txt\": \"I feel Blockchain technologies have been over-hyped.\",\n",
            "  \"statement_id\": 1,\n",
            "  \"created\": \"2018-12-11T16:13:43.605000Z\",\n",
            "  \"tweet_id\": null,\n",
            "  \"quote_src_url\": null,\n",
            "  \"is_seed\": false,\n",
            "  \"is_meta\": false,\n",
            "  \"lang\": \"en\",\n",
            "  \"participant_id\": 0,\n",
            "  \"velocity\": 1,\n",
            "  \"moderated\": -1,\n",
            "  \"active\": true,\n",
            "  \"agree_count\": 12,\n",
            "  \"disagree_count\": 2,\n",
            "  \"pass_count\": 1,\n",
            "  \"count\": 15,\n",
            "  \"conversation_id\": \"6jrufhr6dp\",\n",
            "  \"datetime\": null\n",
            "}\n"
          ]
        }
      ],
      "source": [
        "from reddwarf.data_loader import Loader\n",
        "import json\n",
        "import numpy as np\n",
        "\n",
        "# Topic: What were the most significant developments in tech and politics in 2018?\n",
        "# 5 groups, 65 ptpts (56 grouped), 43 comments (open)\n",
        "# REPORT_ID=\"r7cwmiaxczyj8te9rzdmx\" # DemDis\n",
        "# REPORT_ID = \"r29kkytnipymd3exbynkd\"  # chiledesperto\n",
        "# REPORT_ID=\"r6ipxzfudddppwesbmtmn\" # Alternative Polis convo\n",
        "\n",
        "# Topic: What were the most significant developments in tech and politics in 2018?\n",
        "REPORT_ID = \"r2dfw8eambusb8buvecjt\"\n",
        "print(f\"Loading data from https://pol.is/report/{REPORT_ID}\")\n",
        "\n",
        "# Using only the conversation ID, to demonstrate that export CSV is not used\n",
        "CONVO_ID = \"6jrufhr6dp\"\n",
        "\n",
        "# We'll use the data_loader utility to simply load vote data.\n",
        "loader = Loader(polis_id=CONVO_ID, data_source=\"api\")\n",
        "loader.load_api_data_math()\n",
        "votes = loader.votes_data\n",
        "\n",
        "# Show what our raw vote data looks like:\n",
        "print(json.dumps(votes[0], indent=2))\n",
        "\n",
        "# Show what our raw statement data looks like:\n",
        "statements = loader.comments_data\n",
        "print(json.dumps(statements[0], indent=2))\n",
        "\n",
        "# We need this for a little helper later.\n",
        "math_data = loader.math_data"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Export partial data export in polis CSV format\n",
        "loader.export_data(f\"exports/{loader.conversation_id}/\")"
      ],
      "metadata": {
        "id": "mYGkUHlaQwBs"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Expected working files: comments.csv, participant-votes.csv, summary.csv, votes.csv\n",
        "# Expected empty files: comment-groups.csv\n",
        "!ls exports/*/"
      ],
      "metadata": {
        "id": "-8mlob8-RmYo",
        "outputId": "855a0273-4fed-47b6-c307-53735e548394",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "comment-groups.csv  comments.csv  participant-votes.csv  summary.csv  votes.csv\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}