This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package main | |
import ( | |
"bufio" | |
"fmt" | |
"os" | |
"github.com/cespare/xxhash" | |
) |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import os | |
import fasttext | |
import re | |
lid = fasttext.load_model("lid.176.bin") | |
def listdir(x): | |
return [x + '/' + fn for fn in os.listdir(x)] | |
def id(x): |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#!/bin/sh | |
set -x | |
# == Swarm training (alpha release) == | |
# Setup: | |
# | |
# git clone https://github.com/shawwn/gpt-2 | |
# cd gpt-2 | |
# git checkout dev-shard |