This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import sqlite3 | |
from argparse import ArgumentError | |
from sqlite3 import Connection | |
from sqlite3.dbapi2 import NotSupportedError | |
from typing import Any, List, Optional, Tuple | |
DB_PATH = "data_out/database.db" | |
class SqlConnection: |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import bz2 | |
import lzma | |
from src.reddit_input_processing.zreader import Zreader | |
def decode_bz2_posts(file_path: str, base_path: str, subreddits: Set[str]): | |
with bz2.BZ2File(f"{base_path}/{file_path}", "rb") as source_file: | |
read_posts(source_file, subreddits) | |
def decode_xz_posts(file_path: str, base_path: str, subreddits: Set[str]): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import argparse | |
parser = argparse.ArgumentParser() | |
parser.add_argument("--type", dest="t", action="store", required=True) | |
save_folder = "./data_in/" | |
start_year = 2008 | |
end_year = 2019 |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static class DbSetInitializer | |
{ | |
public static void InitDb<T>(IQueryable<T> set, IQueryable<T> data) where T : class | |
{ | |
set.Provider.Returns(data.Provider); | |
set.Expression.Returns(data.Expression); | |
set.ElementType.Returns(data.ElementType); | |
set.GetEnumerator().Returns(data.GetEnumerator()); | |
set.AsNoTracking().Returns(data); | |
} |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
public static class AutomapperExtensions | |
{ | |
public static IMappingExpression<TSource, TDestination> Ignore<TSource, TDestination>( | |
this IMappingExpression<TSource, TDestination> map, | |
Expression<Func<TDestination, object>> selector) | |
{ | |
map.ForMember(selector, options => options.Ignore()); | |
return map; | |
} | |
public static IMappingExpression<TSource, TDestination> Define<TSource, TDestination>( |