R/chunk_mapper.r
create_chunk_mapper.RdA function to make it easier to create functions like filter
create_chunk_mapper(chunk_fn, warning_msg = NULL, as.data.frame = FALSE)The dplyr function to create a mapper for
The warning message to display when invoking the mapper
force the input chunk of a data.frame; needed for dtplyr
filter = create_chunk_mapper(dplyr::filter)
#' example: creating a function that keeps only the first and last n row
first_and_last <- function(chunk, n, ...) {
nr = nrow(chunk)
print(nr-n+1:nr)
chunk[c(1:n, (nr-n+1):nr), ]
}
#' create the function for use with disk.frame
first_and_last_df = create_chunk_mapper(first_and_last)
mtcars.df = as.disk.frame(mtcars)
#' the operation is lazy
lazy_mtcars.df = mtcars.df %>%
first_and_last_df(2)
#' bring into R
collect(lazy_mtcars.df)
#> [1] 5 6 7 8 9 10
#> [1] 5 6 7 8 9 10
#> [1] 5 6 7 8 9 10
#> [1] 5 6 7 8 9 10
#> [1] 5 6 7 8 9 10
#> [1] 1 2
#> mpg cyl disp hp drat wt qsec vs am gear carb
#> 1: 21.0 6 160.0 110 3.90 2.620 16.46 0 1 4 4
#> 2: 21.0 6 160.0 110 3.90 2.875 17.02 0 1 4 4
#> 3: 18.7 8 360.0 175 3.15 3.440 17.02 0 0 3 2
#> 4: 18.1 6 225.0 105 2.76 3.460 20.22 1 0 3 1
#> 5: 14.3 8 360.0 245 3.21 3.570 15.84 0 0 3 4
#> 6: 24.4 4 146.7 62 3.69 3.190 20.00 1 0 4 2
#> 7: 17.8 6 167.6 123 3.92 3.440 18.90 1 0 4 4
#> 8: 16.4 8 275.8 180 3.07 4.070 17.40 0 0 3 3
#> 9: 17.3 8 275.8 180 3.07 3.730 17.60 0 0 3 3
#> 10: 15.2 8 275.8 180 3.07 3.780 18.00 0 0 3 3
#> 11: 14.7 8 440.0 230 3.23 5.345 17.42 0 0 3 4
#> 12: 32.4 4 78.7 66 4.08 2.200 19.47 1 1 4 1
#> 13: 30.4 4 75.7 52 4.93 1.615 18.52 1 1 4 2
#> 14: 33.9 4 71.1 65 4.22 1.835 19.90 1 1 4 1
#> 15: 15.2 8 304.0 150 3.15 3.435 17.30 0 0 3 2
#> 16: 13.3 8 350.0 245 3.73 3.840 15.41 0 0 3 4
#> 17: 19.2 8 400.0 175 3.08 3.845 17.05 0 0 3 2
#> 18: 27.3 4 79.0 66 4.08 1.935 18.90 1 1 4 1
#> 19: 15.8 8 351.0 264 4.22 3.170 14.50 0 1 5 4
#> 20: 19.7 6 145.0 175 3.62 2.770 15.50 0 1 5 6
#> 21: 15.0 8 301.0 335 3.54 3.570 14.60 0 1 5 8
#> 22: 21.4 4 121.0 109 4.11 2.780 18.60 1 1 4 2
#> 23: 15.0 8 301.0 335 3.54 3.570 14.60 0 1 5 8
#> 24: 21.4 4 121.0 109 4.11 2.780 18.60 1 1 4 2
#> mpg cyl disp hp drat wt qsec vs am gear carb
#' clean up
delete(mtcars.df)