muellerzr

[project]
name = "my_project"
version = "0.1.0"
description = "A simple Python project"
authors = [
  { name = "Your Name", email = "[email protected]" }
]
readme = "README.md"
license = { text = "MIT" }
requires-python = ">=3.8"

muellerzr

import torch
from torch.utils.data import DataLoader
from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling
from datasets import load_from_disk
import torch.nn.functional as F
from torch.nn import CrossEntropyLoss
import numpy as np
import logging
import time
from datetime import datetime

muellerzr

import json
import os
from datetime import datetime
import argparse
from typing import Dict, List, Optional

"""
Quick Journal is a simple CLI journal that allows you to write, search, and export journal entries.
Specialized for ML experiment tracking with predefined fields.

muellerzr

import torch.nn as nn
from datasets import load_dataset
from transformers import (
    AutoModelForCausalLM,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    Trainer,
    TrainingArguments,
    set_seed,
)

muellerzr

# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,

muellerzr

muellerzr

compute_environment: LOCAL_MACHINE
debug: false
distributed_type: FSDP
downcast_bf16: 'no'
enable_cpu_affinity: false
fsdp_config:
  fsdp_activation_checkpointing: false
  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
  fsdp_backward_prefetch: BACKWARD_PRE
  fsdp_cpu_ram_efficient_loading: true

muellerzr

import time
from transformers import AutoTokenizer, LlamaForCausalLM
from accelerate.utils import set_seed

set_seed(42)


file_size = 132 # 70B
# file_size = 30 # 8B
start_time = time.time()

muellerzr

name: Deploy to GitHub Pages
on:
  push:
    branches: [ "main", "master" ]
  workflow_dispatch:
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

muellerzr

[P2P (Peer-to-Peer) GPU Bandwidth Latency Test]
Device: 0, NVIDIA GeForce RTX 4090, pciBusID: 1, pciDeviceID: 0, pciDomainID:0
Device: 1, NVIDIA GeForce RTX 4090, pciBusID: 2, pciDeviceID: 0, pciDomainID:0
Device=0 CANNOT Access Peer Device=1
Device=1 CANNOT Access Peer Device=0

***NOTE: In case a device doesn't have P2P access to other one, it falls back to normal memcopy procedure.
So you can see lesser Bandwidth (GB/s) and unstable Latency (us) in those cases.

P2P Connectivity Matrix