This will materialize a lazy_tensor()
or a data.frame()
/ list()
containing – among other things –
lazy_tensor()
columns.
I.e. the data described in the underlying DataDescriptor
s is loaded for the indices in the lazy_tensor()
,
is preprocessed and then put unto the specified device.
Because not all elements in a lazy tensor must have the same shape, a list of tensors is returned by default.
If all elements have the same shape, these tensors can also be rbinded into a single tensor (parameter rbind
).
Usage
materialize(x, device = "cpu", rbind = FALSE, ...)
# S3 method for class 'list'
materialize(x, device = "cpu", rbind = FALSE, cache = "auto", ...)
Arguments
- x
(any)
The object to materialize. Either alazy_tensor
or alist()
/data.frame()
containinglazy_tensor
columns.- device
(
character(1)
)
The torch device.- rbind
(
logical(1)
)
Whether to rbind the lazy tensor columns (TRUE
) or return them as a list of tensors (FALSE
). In the second case, there is no batch dimension.- ...
(any)
Additional arguments.- cache
(
character(1)
orenvironment()
orNULL
)
Optional cache for (intermediate) materialization results. Per default, caching will be enabled when the same dataset or data descriptor (with different output pointer) is used for more than one lazy tensor column.
Value
(list()
of lazy_tensor
s or a lazy_tensor
)
Details
Materializing a lazy tensor consists of:
Loading the data from the internal dataset of the
DataDescriptor
.Processing these batches in the preprocessing
Graph
s.Returning the result of the
PipeOp
pointed to by theDataDescriptor
(pointer
).
With multiple lazy_tensor
columns we can benefit from caching because:
a) Output(s) from the dataset might be input to multiple graphs.
b) Different lazy tensors might be outputs from the same graph.
For this reason it is possible to provide a cache environment. The hash key for a) is the hash of the indices and the dataset. The hash key for b) is the hash of the indices, dataset and preprocessing graph.
Examples
lt1 = as_lazy_tensor(torch_randn(10, 3))
materialize(lt1, rbind = TRUE)
#> torch_tensor
#> -0.5175 2.1826 0.5440
#> 0.6128 2.2153 1.6310
#> 0.8558 -0.3944 0.4699
#> -0.3781 -0.5245 -1.1661
#> 0.2729 0.1567 1.2514
#> -0.9974 -1.6371 -0.7166
#> 0.7729 -1.2926 0.9860
#> -0.6191 1.6560 0.6337
#> 1.5480 0.1169 -0.4474
#> -0.0400 -2.8373 0.2412
#> [ CPUFloatType{10,3} ]
materialize(lt1, rbind = FALSE)
#> [[1]]
#> torch_tensor
#> -0.5175
#> 2.1826
#> 0.5440
#> [ CPUFloatType{3} ]
#>
#> [[2]]
#> torch_tensor
#> 0.6128
#> 2.2153
#> 1.6310
#> [ CPUFloatType{3} ]
#>
#> [[3]]
#> torch_tensor
#> 0.8558
#> -0.3944
#> 0.4699
#> [ CPUFloatType{3} ]
#>
#> [[4]]
#> torch_tensor
#> -0.3781
#> -0.5245
#> -1.1661
#> [ CPUFloatType{3} ]
#>
#> [[5]]
#> torch_tensor
#> 0.2729
#> 0.1567
#> 1.2514
#> [ CPUFloatType{3} ]
#>
#> [[6]]
#> torch_tensor
#> -0.9974
#> -1.6371
#> -0.7166
#> [ CPUFloatType{3} ]
#>
#> [[7]]
#> torch_tensor
#> 0.7729
#> -1.2926
#> 0.9860
#> [ CPUFloatType{3} ]
#>
#> [[8]]
#> torch_tensor
#> -0.6191
#> 1.6560
#> 0.6337
#> [ CPUFloatType{3} ]
#>
#> [[9]]
#> torch_tensor
#> 1.5480
#> 0.1169
#> -0.4474
#> [ CPUFloatType{3} ]
#>
#> [[10]]
#> torch_tensor
#> -0.0400
#> -2.8373
#> 0.2412
#> [ CPUFloatType{3} ]
#>
lt2 = as_lazy_tensor(torch_randn(10, 4))
d = data.table::data.table(lt1 = lt1, lt2 = lt2)
materialize(d, rbind = TRUE)
#> $lt1
#> torch_tensor
#> -0.5175 2.1826 0.5440
#> 0.6128 2.2153 1.6310
#> 0.8558 -0.3944 0.4699
#> -0.3781 -0.5245 -1.1661
#> 0.2729 0.1567 1.2514
#> -0.9974 -1.6371 -0.7166
#> 0.7729 -1.2926 0.9860
#> -0.6191 1.6560 0.6337
#> 1.5480 0.1169 -0.4474
#> -0.0400 -2.8373 0.2412
#> [ CPUFloatType{10,3} ]
#>
#> $lt2
#> torch_tensor
#> -0.6276 -0.1095 -0.7908 0.5330
#> -0.0547 -0.1122 -0.4323 0.3639
#> 1.2063 -0.8808 0.5885 -0.8140
#> -0.2477 -0.2911 0.4360 -1.2602
#> -0.4050 -1.3976 0.5262 -0.6855
#> -1.1056 1.2594 -0.3555 1.7537
#> -1.5813 1.0822 1.9626 2.1383
#> -0.8657 -0.8745 0.8644 -0.8650
#> 1.9690 -0.4740 -0.1647 -1.4104
#> 0.2607 0.9733 -1.1057 0.5108
#> [ CPUFloatType{10,4} ]
#>
materialize(d, rbind = FALSE)
#> $lt1
#> $lt1[[1]]
#> torch_tensor
#> -0.5175
#> 2.1826
#> 0.5440
#> [ CPUFloatType{3} ]
#>
#> $lt1[[2]]
#> torch_tensor
#> 0.6128
#> 2.2153
#> 1.6310
#> [ CPUFloatType{3} ]
#>
#> $lt1[[3]]
#> torch_tensor
#> 0.8558
#> -0.3944
#> 0.4699
#> [ CPUFloatType{3} ]
#>
#> $lt1[[4]]
#> torch_tensor
#> -0.3781
#> -0.5245
#> -1.1661
#> [ CPUFloatType{3} ]
#>
#> $lt1[[5]]
#> torch_tensor
#> 0.2729
#> 0.1567
#> 1.2514
#> [ CPUFloatType{3} ]
#>
#> $lt1[[6]]
#> torch_tensor
#> -0.9974
#> -1.6371
#> -0.7166
#> [ CPUFloatType{3} ]
#>
#> $lt1[[7]]
#> torch_tensor
#> 0.7729
#> -1.2926
#> 0.9860
#> [ CPUFloatType{3} ]
#>
#> $lt1[[8]]
#> torch_tensor
#> -0.6191
#> 1.6560
#> 0.6337
#> [ CPUFloatType{3} ]
#>
#> $lt1[[9]]
#> torch_tensor
#> 1.5480
#> 0.1169
#> -0.4474
#> [ CPUFloatType{3} ]
#>
#> $lt1[[10]]
#> torch_tensor
#> -0.0400
#> -2.8373
#> 0.2412
#> [ CPUFloatType{3} ]
#>
#>
#> $lt2
#> $lt2[[1]]
#> torch_tensor
#> -0.6276
#> -0.1095
#> -0.7908
#> 0.5330
#> [ CPUFloatType{4} ]
#>
#> $lt2[[2]]
#> torch_tensor
#> -0.0547
#> -0.1122
#> -0.4323
#> 0.3639
#> [ CPUFloatType{4} ]
#>
#> $lt2[[3]]
#> torch_tensor
#> 1.2063
#> -0.8808
#> 0.5885
#> -0.8140
#> [ CPUFloatType{4} ]
#>
#> $lt2[[4]]
#> torch_tensor
#> -0.2477
#> -0.2911
#> 0.4360
#> -1.2602
#> [ CPUFloatType{4} ]
#>
#> $lt2[[5]]
#> torch_tensor
#> -0.4050
#> -1.3976
#> 0.5262
#> -0.6855
#> [ CPUFloatType{4} ]
#>
#> $lt2[[6]]
#> torch_tensor
#> -1.1056
#> 1.2594
#> -0.3555
#> 1.7537
#> [ CPUFloatType{4} ]
#>
#> $lt2[[7]]
#> torch_tensor
#> -1.5813
#> 1.0822
#> 1.9626
#> 2.1383
#> [ CPUFloatType{4} ]
#>
#> $lt2[[8]]
#> torch_tensor
#> -0.8657
#> -0.8745
#> 0.8644
#> -0.8650
#> [ CPUFloatType{4} ]
#>
#> $lt2[[9]]
#> torch_tensor
#> 1.9690
#> -0.4740
#> -0.1647
#> -1.4104
#> [ CPUFloatType{4} ]
#>
#> $lt2[[10]]
#> torch_tensor
#> 0.2607
#> 0.9733
#> -1.1057
#> 0.5108
#> [ CPUFloatType{4} ]
#>
#>