master_data_science/06-advanced_import_export.R at master · joseramoncajide/master_data_science · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
##########################################################################
# Jose Cajide - @jrcajide
# Master Data Science: Advanced data reading and writing
##########################################################################


require(readr)  # for read_csv()
require(dplyr)  # for mutate()
require(tidyr)  # for unnest()
require(purrr)  # for map(), reduce()
require(data.table) # for fread()


# Reading data ------------------------------------------------------------


data_path <- file.path("data", "flights")
files <- dir(data_path, pattern = "*.csv")


flights <- files %>%
  # read in all the files, appending the path before the filename
  map(~ read_csv(file.path(data_path, .))) %>%
  reduce(rbind)
flights

# The same using an anonymous function
flights <- files %>%
  map(function(x) read_csv(file.path(data_path, x))) %>%
  reduce(rbind)
flights


system.time( flights <- data_frame(filename = files) %>% # create a data frame
               # holding the file names
               mutate(file_contents = map(filename,          # read files into
                                          ~ data.table::fread(file.path(data_path, .), showProgress=T, nThread=4)) # a new data column
               )  )

flights

flights <- unnest(flights)

flights

print(object.size(get('flights')), units='auto')


# Exporting data ----------------------------------------------------------

flights %>%
  sample_n(1000) %>%
  write_csv(., file.path("exports", "flights.csv"))


flights %>%
  group_by(Month) %>%
  do(tail(., 2))

dir.create('exports')

flights %>%
  sample_n(1000) %>%
  group_by(Year, Month) %>%
  do(write_csv(., file.path("exports", paste0(unique(.$Year),"_",unique(.$Month), "_flights.csv"))))