You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
13 lines
374 B
13 lines
374 B
2 weeks ago
|
from tinygrad import Tensor, GlobalCounters, Context
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
with Context(TRACK_MATCH_STATS=0): test = Tensor.ones(32, 10).contiguous().realize()
|
||
|
|
GlobalCounters.reset()
|
||
|
|
|
||
|
|
# this is the softmax from scaled_dot_product_attention
|
||
|
|
# it becomes 3 kernels
|
||
|
|
print("*** softmax ***")
|
||
|
|
with Context(NOOPT=1):
|
||
|
|
out = test.softmax(-1)
|
||
|
|
out.realize()
|