| import time | |
| import numpy as np | |
| import torch | |
| from itertools import accumulate | |
| from flash_attn import flash_attn_varlen_qkvpacked_func, flash_attn_qkvpacked_func | |
| device = torch.device('cuda:0') | |
| dtype = torch.bfloat16 |
People
 :bowtie: |
😄 :smile: |
😆 :laughing: |
|---|---|---|
😊 :blush: |
😃 :smiley: |
:relaxed: |
😏 :smirk: |
😍 :heart_eyes: |
😘 :kissing_heart: |
😚 :kissing_closed_eyes: |
😳 :flushed: |
😌 :relieved: |
😆 :satisfied: |
😁 :grin: |
😉 :wink: |
😜 :stuck_out_tongue_winking_eye: |
😝 :stuck_out_tongue_closed_eyes: |
😀 :grinning: |
😗 :kissing: |
😙 :kissing_smiling_eyes: |
😛 :stuck_out_tongue: |
| # This isn't supposed to run as a bash script, i named it with ".sh" for syntax highlighting. | |
| # https://developer.nvidia.com/nsight-systems | |
| # https://docs.nvidia.com/nsight-systems/profiling/index.html | |
| # My preferred nsys (command line executable used to create profiles) commands | |
| # | |
| # In your script, write | |
| # torch.cuda.nvtx.range_push("region name") | |
| # ... |
| import torch | |
| import torch.nn as nn | |
| import random | |
| import math | |
| def hinge_loss(scores, targets): | |
| ''' | |
| multi-class extension: https://en.wikipedia.org/wiki/Hinge_loss | |
| Weston Jason & Watkins Chris (1999). | |
| Support Vector Machines for Multi-Class Pattern Recognition. |
