Aphoh

(run_docker pid=4406, ip=10.164.0.48)   File "/opt/levanter/.venv/lib/python3.10/site-packages/jax/_src/compiler.py", line 261, in backend_compile
(run_docker pid=4406, ip=10.164.0.48)     return backend.compile(
(run_docker pid=4406, ip=10.164.0.48) jaxlib.xla_extension.XlaRuntimeError: INTERNAL: Mosaic failed to compile TPU kernel: Unsupported input data type in matrix multiplication in this target.
(run_docker pid=4406, ip=10.164.0.48) 
(run_docker pid=4406, ip=10.164.0.48) at location: loc("/dot_general"(callsite("_splash_attention"("/opt/levanter/.venv/lib/python3.10/site-packages/jax/experimental/pallas/ops/tpu/splash_attention/splash_attention_kernel.py":2277:0) at callsite("__call__"("/opt/levanter/.venv/lib/python3.10/site-packages/jax/experimental/pallas/ops/tpu/splash_attention/splash_attention_kernel.py":2312:0) at callsite("<lambda>"("/opt/levanter/src/levanter/models/attention.py":953:0) at callsite("wrap_flash_attention"("/opt/levanter/src/levanter/models/attention.py":952:0) at callsite("_tpu_

Aphoh

2025-01-23T13:34:40 - 0 - tqdm_loggable.tqdm_logging - tqdm_logging.py:145 - INFO :: Progress on:train 112it/1.00kit rate:84.6s/it remaining:20:51:22 elapsed:2:16:28 postfix:loss=0.698
318 Traceback (most recent call last):
319   File "/opt/levanter/src/levanter/main/routed_lm.py", line 290, in <module>
320     levanter.config.main(main)()
321   File "/opt/levanter/src/levanter/config.py", line 84, in wrapper_inner
322     response = fn(cfg, *args, **kwargs)
323   File "/opt/levanter/src/levanter/main/routed_lm.py", line 277, in main
324     last_info = trainer.train(state, train_loader)
325   File "/opt/levanter/src/levanter/trainer.py", line 428, in train
326     for info in self.training_steps(state, train_loader):

Aphoh

Turning off:
IRSend3 raw,0,8000,4200,525,525,1580,110000000100000011111111000110010000000000011000

Turning on:
IRSend3 raw,0,8000,4200,525,525,1580,110100000100000011111111000110011000000010101000

off: 110000000100000011111111000110010000000000011000
on:  110100000100000011111111000110011000000010101000

Aphoh

Aphoh

   //Feast your eyes on the worst pointer chasing logic I have ever seen.
   static simulation_context* ctx_from_term(ErlNifEnv *env, ERL_NIF_TERM term) {
   
    /* This may seem crazy, but here's what's going on here. 
     * I'm trying to get a simulation_context*, which is a resource
     * my application is using. The function enif_get_resource accepts a void** ptr such that *ptr           
     * will point to my resource, so **ptr will be my simulation_context*
     */
    
    // We can't make a simulation_context*** as then it will try to write the pointer-pointer to that location

Aphoh

##### Replace 'example' anywhere with the name of your app and '*ec2ip*' with your ec2 instance ip
##### Set up your instance and make sure it's security group has ssh, http, and https open inbound and outbound
##### Don't forget to chmod 400 cert.pem

##### .deliver/config
APP="example"

BUILD_HOST="*ec2ip*"
BUILD_USER="elixir_builder"
BUILD_AT="/home/$BUILD_USER/edeliver/$APP/builds"

Aphoh

// High quality
// Price range 1
gen   best_one  = .

set trace on

mkmat *_5_1, matrix(food_5_1)


forvalues i=1/56 {