Pytorch Tensor Tutorial
Contents
Pytorch Tensor Tutorial¶
Initializing tensors, math, indexing, reshaping
Tensor Initialization¶
import torch
# create a tensor
device = "cuda" if torch.cuda.is_available() else "cpu" # if we have a GPU then use it
my_tensor = torch.tensor([[1, 2, 3], [4, 5, 6]], # 2 rows and 3 columns
dtype=torch.float32, # float32 type
device=device, requires_grad=True) # default device = "cpu"
print(my_tensor)
tensor([[1., 2., 3.],
[4., 5., 6.]], requires_grad=True)
We can see the attributes of a tensor like below, which we have defined when initialiazation.
print(my_tensor.dtype)
print(my_tensor.device)
print(my_tensor.shape)
print(my_tensor.requires_grad)
torch.float32
cpu
torch.Size([2, 3])
True
Other initialization methods:
x = torch.empty(size = (3, 3)) # random values
print(x)
tensor([[ 0.0000e+00, -1.0842e-19, 7.2784e-17],
[-2.0005e+00, 2.3594e+00, 4.5898e-41],
[ 3.9081e+00, 4.5898e-41, 3.9110e+19]])
x = torch.zeros((3, 3)) # zeros values
print(x)
tensor([[0., 0., 0.],
[0., 0., 0.],
[0., 0., 0.]])
x = torch.rand((3, 3)) # random values in [0,1]
print(x)
tensor([[0.6710, 0.7784, 0.0981],
[0.2184, 0.1089, 0.1406],
[0.6998, 0.3420, 0.0784]])
x = torch.ones((3, 3)) # ones values
print(x)
tensor([[1., 1., 1.],
[1., 1., 1.],
[1., 1., 1.]])
x = torch.eye(5, 5) # identity matrix
print(x)
tensor([[1., 0., 0., 0., 0.],
[0., 1., 0., 0., 0.],
[0., 0., 1., 0., 0.],
[0., 0., 0., 1., 0.],
[0., 0., 0., 0., 1.]])
x = torch.arange(start=0, end=5, step=1) # range[0,5]
print(x)
tensor([0, 1, 2, 3, 4])
x = torch.linspace(start=0.1, end=1, steps=10) # equal cut into parts
print(x)
tensor([0.1000, 0.2000, 0.3000, 0.4000, 0.5000, 0.6000, 0.7000, 0.8000, 0.9000,
1.0000])
x = torch.empty(size = (1, 5)).normal_(mean=0, std=1) # normal distribution
print(x)
tensor([[-0.2402, -1.6307, 0.6818, -0.0020, 1.0872]])
x = torch.empty(size = (1, 5)).uniform_(0, 1) # uniform distribution, same as torch.rand()
print(x)
tensor([[0.4704, 0.5660, 0.3666, 0.4855, 0.4403]])
x = torch.diag(torch.ones(3)) # diagonal matrix with ones values, same as torch.eye(3, 3)
print(x)
tensor([[1., 0., 0.],
[0., 1., 0.],
[0., 0., 1.]])
How to initialize and convert tensors to other types (int, float, double)
# create an int64 tensor
tensor = torch.arange(4)
# convert it into a boolean tensor
print(tensor.bool())
# convert to int16 tensor
print(tensor.short())
# convert to int64 tensor
print(tensor.long()) # important to use
# convert to float16 tensor
print(tensor.half())
# convert to float32 tensor
print(tensor.float()) # most often to use
# convert to float64 tensor
print(tensor.double())
tensor([False, True, True, True])
tensor([0, 1, 2, 3], dtype=torch.int16)
tensor([0, 1, 2, 3])
tensor([0., 1., 2., 3.], dtype=torch.float16)
tensor([0., 1., 2., 3.])
tensor([0., 1., 2., 3.], dtype=torch.float64)
Array to Tensor conversion and vice-versa
import numpy as np
np_array = np.zeros((5, 5))
# array to tensor
tensor = torch.from_numpy(np_array)
# tensor to array
np_array_back = tensor.numpy()
print(tensor)
print(np_array_back)
tensor([[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.]], dtype=torch.float64)
[[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]
[0. 0. 0. 0. 0.]]
Tensor Math & Comparison Operations¶
x = torch.tensor([1, 2, 3])
y = torch.tensor([9, 8, 7])
# addition
z1 = torch.empty(3)
torch.add(x, y, out=z1)
print(z1)
# same result
z2 = torch.add(x, y)
print(z2)
# cleanest method
z = x + y
print(z)
tensor([10., 10., 10.])
tensor([10, 10, 10])
tensor([10, 10, 10])
# Substraction
z = x - y
print(z)
tensor([-8, -6, -4])
# Division
z = torch.true_divide(x, y) # equal shape of x and y
print(z)
tensor([0.1111, 0.2500, 0.4286])
# inplace operations
t = torch.zeros(3)
t.add_(x)
tensor([1., 2., 3.])
t += x # t = t + x
print(x)
tensor([1, 2, 3])
# Exponentiation
z = x.pow(2) # a power of 2 # same as z = x ** 2
print(z)
tensor([1, 4, 9])
# Simple comparison
z = x > 0
print(z)
tensor([True, True, True])
# Matrix Multiplication
x1 = torch.rand((2, 5))
x2 = torch.rand((5, 3))
x3 = torch.mm(x1, x2) # matrix 2x3
# same result
x3 = x1.mm(x2)
print(x3)
tensor([[1.6430, 0.5692, 1.2012],
[1.3496, 0.6505, 0.9740]])
# Matrix exponentiation
matrix_exp = torch.rand(5, 5)
matrix_exp.matrix_power(3) # matrix A^3
tensor([[0.7119, 0.6393, 0.8152, 1.2186, 0.6486],
[1.6174, 1.0445, 1.5734, 2.3789, 1.6560],
[0.8598, 0.6528, 0.8238, 1.3173, 0.8435],
[1.1377, 0.8835, 1.2167, 1.7583, 0.9608],
[0.9672, 0.4661, 0.9198, 1.2707, 1.0546]])
# element wise multiplication
z = x * y
print(z)
tensor([ 9, 16, 21])
# dot product
z = torch.dot(x, y)
print(z)
tensor(46)
# Batch Matrix Multiplication
batch = 32
n = 10
m = 20
p = 30
tensor1 = torch.rand((batch, n, m))
tensor2 = torch.rand((batch, m, p))
out_bmm = torch.bmm(tensor1, tensor2) # shape is (batch, n, p)
# Example of Broadcasting
x1 = torch.rand((5, 5)) # matrix
x2 = torch.rand((1, 5)) # vector
z = x1 - x2
print(z)
tensor([[-0.1588, 0.6100, 0.5474, 0.0453, 0.2111],
[ 0.1421, 0.5433, 0.3209, -0.5023, 0.0719],
[ 0.1993, 0.3626, 0.5996, -0.0093, 0.2360],
[-0.5452, 0.8181, 0.9098, 0.0167, 0.6901],
[-0.6668, 0.7660, 0.7309, -0.5631, 0.5419]])
# other useful tensor operations
sum_x = torch.sum(x, dim=0) # which dimension used to sum over
values, indices = torch.max(x, dim=0) # x.max(dim=0)
values, indices = torch.min(x, dim=0) # x.min(dim=0)
abs_x = torch.abs(x)
z = torch.argmax(x, dim=0)
z = torch.argmin(x, dim=0)
mean_x = torch.mean(x.float(), dim=0)
z = torch.eq(x, y) # which elements are the same
print(z)
tensor([False, False, False])
torch.sort(y, dim=0, descending=False) # return values and indices
torch.return_types.sort(
values=tensor([7, 8, 9]),
indices=tensor([2, 1, 0]))
z = torch.clamp(x, min=2, max=2) # 将不在范围的值替换为最大/最小值
print(x)
print(z)
tensor([1, 2, 3])
tensor([2, 2, 2])
x = torch.tensor([1,0,1,1,1], dtype = torch.bool)
z1 = torch.any(x) # or
z2 = torch.all(x) # and
print(z1, z2)
tensor(True) tensor(False)
Tensor Indexing¶
# indexing
batch_size = 10
features = 25
x = torch.rand((batch_size, features))
print(x[0].shape) # x[0,:]
torch.Size([25])
print(x[:,0].shape) # x[:,0]
torch.Size([10])
print(x[2, 0:10]) # third row, first 10 columns
tensor([0.8051, 0.0079, 0.5760, 0.6320, 0.6954, 0.4253, 0.0131, 0.3714, 0.6121,
0.5631])
x[0,0] = 100
print(x[0])
tensor([1.0000e+02, 9.8690e-01, 6.6475e-01, 1.8049e-01, 8.7535e-02, 7.2265e-01,
5.3757e-01, 4.9548e-01, 7.3308e-01, 3.1217e-01, 2.1519e-01, 6.7550e-01,
3.7073e-01, 6.6886e-01, 8.3677e-01, 1.1607e-01, 1.6496e-01, 8.8467e-01,
4.4436e-01, 9.0770e-01, 3.4327e-01, 8.8100e-01, 9.8578e-01, 4.0607e-01,
9.9166e-01])
# Fancy indexing
x = torch.arange(10)
indices = [2,5,8]
print(x[indices])
tensor([2, 5, 8])
x = torch.rand((3, 5))
rows = torch.tensor([1, 0])
cols = torch.tensor([4, 0])
print(x[rows, cols].shape) # pick out the element of (1,4) and (0,0)
torch.Size([2])
# More advanced indexing
x = torch.arange(10)
print(x[(x < 2) | (x > 8)]) # pick out smaller than 2 or bigger than 8
print(x[(x < 2) & (x > 8)]) # pick out smaller than 2 and bigger than 8
tensor([0, 1, 9])
tensor([], dtype=torch.int64)
print(x[x.remainder(2) == 0]) # remainder == 0
tensor([0, 2, 4, 6, 8])
# useful opertions
print(torch.where(x > 5, x, x*2)) # same as np.where()
tensor([ 0, 2, 4, 6, 8, 10, 6, 7, 8, 9])
print(torch.tensor([0,0,1,2,2,3,4]).unique()) # return unique elements
tensor([0, 1, 2, 3, 4])
print(x.ndimension()) # the number of dimensions
1
print(x.numel()) # number of elements
10
Tensor Reshaping¶
x = torch.arange(9)
x_3x3 = x.view(3, 3)
x_3x3 = x.reshape(3, 3)
print(x_3x3)
tensor([[0, 1, 2],
[3, 4, 5],
[6, 7, 8]])
y = x_3x3.t() # transpose
print(y)
print(y.reshape(9))
print(y.contiguous().view(9))
tensor([[0, 3, 6],
[1, 4, 7],
[2, 5, 8]])
tensor([0, 3, 6, 1, 4, 7, 2, 5, 8])
tensor([0, 3, 6, 1, 4, 7, 2, 5, 8])
x1 = torch.rand((2, 5))
x2 = torch.rand((2, 5))
print(torch.cat((x1, x2), dim=0).shape) # combine 2 tensors by columns
print(torch.cat((x1, x2), dim=1).shape) # combine 2 tensors by rows
torch.Size([4, 5])
torch.Size([2, 10])
z = x1.view(-1) # number of elements
print(z.shape)
torch.Size([10])
batch = 64
x = torch.rand((batch, 2, 5))
z = x.view(batch, -1) # keep the dimension of `batch`, and conbine other dimensions
print(z.shape)
torch.Size([64, 10])
z = x.permute(0, 2, 1) # change the dimension order: 0 to 0, 2 to 1, 1 to 2
print(z.shape)
torch.Size([64, 5, 2])
x = torch.arange(10) # [10]
print(x.shape)
print(x.unsqueeze(0).shape) # insert 1 at the 0 position of dimension
print(x.unsqueeze(1).shape) # insert 1 at the 1 position of dimension
torch.Size([10])
torch.Size([1, 10])
torch.Size([10, 1])
x = torch.arange(10).unsqueeze(0).unsqueeze(2) # 1x10x1
print(x.shape)
torch.Size([1, 10, 1])
z = x.squeeze(2) # delete the dimension of 1 at the position 2
print(z.shape)
torch.Size([1, 10])