huseinzol05

huseinzol05

huseinzol05

import torch
import torch.nn.functional as F
from torch.nn.attention.flex_attention import flex_attention, create_block_mask
import torch

head_num = 16
dim = 128
seq_len = 100
chunk_size = 5
batch_size = 1

huseinzol05

# train_grpo.py
import re
import torch
from datasets import load_dataset, Dataset
from transformers import AutoTokenizer, AutoModelForCausalLM
from peft import LoraConfig
from trl import GRPOConfig, GRPOTrainer

# Load and prep dataset

huseinzol05

from glob import glob
from tqdm import tqdm
from multiprocess import Pool
import itertools
import zipfile
import os

def chunks(l, n):
    for i in range(0, len(l), n):
        yield (l[i: i + n], i // n)

huseinzol05

import json
import requests
import os
import boto3
import shutil
from tqdm import tqdm

client = boto3.client(
    'bedrock-runtime', 
    region_name='us-west-2',

huseinzol05

git clone https://github.com/nvidia/nccl-tests && cd nccl-tests
sudo apt-get install openmpi-bin openmpi-common libopenmpi-dev -y
make MPI=1 MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi
./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4
mpirun -np 10 -H 10.224.0.47,10.224.0.70 ./build/all_reduce_perf -b 8 -e 128M -f 2 -g 4

huseinzol05

huseinzol05

huseinzol05