|python|PyTorch| |---|---| |Int|IntTensor of size()| |float|FloatTensor of size()| |Int array|IntTensor of size \[d1, d2, ...]| |Float array|FloatTensor of size \[d1, d2, ...]| |string|--| ## How to denote string tensor| |---|---|---|---| |32-bit floating point|torch.float32 or torch.float|torch.FloatTensor|torch.cuda.FloatTensor| |64-bit floating point|torch.float64 or torch.double|torch.DoubleTensor|torch.cuda randn(2, 3) 3 In [6]: a.type() 4 Out[6]: 'torch.FloatTensor' 6 In [7]: type(a) 7 Out[7]: torch.Tensor 9 In [8]: isinstance(a, torch.FloatTensor) 10 Out[8]: True ## ☐ ☐ ☐ 1 In [21]: isinstance(data

are you talking about? ## Example typealias FloatTensor = Tensor func linear(x : FloatTensor, w : FloatTensor, b : FloatTensor) -> FloatTensor { let tmp = w • x let tmp2 = tmp + b 'TensorFlow') (typealias range=[atswift.swift:11:1 - line:11:37] "FloatTensor" interface type='FloatTensor.Type' access=internal type='Tensor(func_decl interface type='(FloatTensor, FloatTensor, FloatTensor) -> float_tensor' parameter_list (parameter "x" apiName=x type='FloatTensor' interface type='FloatTensor') (parameter

● ● 1 In [67]: torch.tensor([2., 3.2]) 2 Out[67]: tensor([2.0000, 3.2000]) 4 In [68]: torch.FloatTensor([2., 3.2]) 5 Out[68]: tensor([2.0000, 3.2000]) 7 In [69]: torch.tensor([[2., 3.2], [1., 8.0] 3.2000], 10 [ 1.0000, 22.3000] ) ## uninitialized • Torch.empty() - Torch.FloatTensor(d1, d2, d3) NOT torch.FloatTensor([1, 2]) = torch.tensor([1, 2]) • Torch.IntTensor(d1, d2, d3) ## uninitialized [ 508, 0, -776122816], 0, -2147483648]], dtype=torch.int32) 14 In [73]: torch.FloatTensor(2,3) 15 Out $$ 73 $$ : 16 tensor([[ 3.1921e+27, 0.0000e+00, -8.0417e-17], 17 [ 7.1186e-43

type(t) or t.type() • returns • numpy.ndarray • torch.Tensor • CPU - torch.cpu.FloatTensor • GPU - torch.cuda.FloatTensor ## Autograd ## • Autograd • Automatic Differentiation Package • Don’t need to

ckend="nccl") local_rank = dist.get_rank() % torch.cuda.device_count() data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group) value = cpu_data (continued from previous page) - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache:

ckend="nccl") local_rank = dist.get_rank() % torch.cuda.device_count() data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group) value = cpu_data (continued from previous page) - past_key_values: Optional[List[torch.FloatTensor]] = None, - inputs_embeds: Optional[torch.FloatTensor] = None, - labels: Optional[torch.LongTensor] = None, - use_cache:

ckend="nccl") local_rank = dist.get_rank() % torch.cuda.device_count() data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group) value = cpu_data Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache:

ckend="nccl") local_rank = dist.get_rank() % torch.cuda.device_count() data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group) value = cpu_data Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache:

ckend="nccl") local_rank = dist.get_rank() % torch.cuda.device_count() data = torch.FloatTensor([1,] * 128).to(f"cuda:{local_rank}") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) dist.all_reduce(cpu_data, op=dist.ReduceOp.SUM, group=gloo_group) value = cpu_data Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache:

= dist.get_rank() % torch.cuda.device_count() torch.cuda.set_device(local_rank) data = torch.FloatTensor([1,] * 128).to("cuda") dist.all_reduce(data, op=dist.ReduceOp.SUM) torch.cuda.synchronize() gloo_group = dist.new_group(ranks=list(range(world_size)), backend="gloo") cpu_data = torch.FloatTensor([1,] * 128) ``` (continues on next page) (continued from previous page) dist.all_reduce(cpu_data Optional[torch.LongTensor] = None, past_key_values: Optional[List[torch.FloatTensor]] = None, inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None, use_cache: