Script to generate labels

label_creation.R

highquality_manual_hq <- function(in_path, out_path, restart = 1) {
  # List all the available points in in_path
  full_points <- list.files(in_path, full.names = TRUE)
  message(sprintf("Hay %s puntos", length(full_points)))
  if (!all(grepl("point_", basename(full_points)))) {
    stop(
      "Existen carpetas o archivos que no empiezan con 'point_', eliminar primero",
      " todos los archivos/carpetas que no esten relacionado con cloudSEN12."
    )
  }
  
  # Create the results folder
  # unlink(out_path, recursive = TRUE) 
  dir.create(sprintf("%s/highquality/manual_hq", out_path), recursive = TRUE, showWarnings = FALSE)
  
  # raster ref 
  raster_ref <- raster(ncol=509, nrow=509, xmn=0, xmx = 5.09, ymn = 0, ymx = 5.09)
  
  for(index in restart:length(full_points)) {
    print(sprintf("PROCESANDO: %s", basename(full_points[index])))
    print(sprintf("INDEX: %s", index))
    manual_files <- list.files(
      path = full_points[index],
      pattern = "manual\\.png",
      full.names = TRUE,
      recursive = TRUE
    )
    
    if (length(manual_files) != 5) {
      stop(
        sprintf(
          "El punto %s tiene solo %s cuando deberia tener %s",
          basename(full_points[index]),
          length(manual_files),
          "5 archivos con el nombre manual.png. REVISAR!!"
        )
      )
    }
    
    new_name <- sprintf(
      "%s/%s__%s__hq.tif",
      out_path,
      basename(full_points[index]),
      basename(dirname(dirname(manual_files)))
    )
    for (index2 in 1:5) {
      hqfinal_tif <- create_tif_hq_final(
        in_path = manual_files[index2],
        raster_ref = raster_ref
      )
      img_id <- basename(dirname(dirname(manual_files[index2])))
      point_id <- basename(dirname(dirname(dirname(manual_files[index2]))))
      writeRaster(
        x = hqfinal_tif, 
        filename = sprintf("%s/highquality/manual_hq/%s__%s.tif", out_path, point_id, img_id),
        overwrite = TRUE
      )
    }    
  }
}

highquality_manual_sc <- function(in_path, out_path, restart = 1) {
  # List all the available points in in_path
  full_points <- list.files(in_path, full.names = TRUE)
  message(sprintf("Hay %s puntos", length(full_points)))
  if (!all(grepl("point_", basename(full_points)))) {
    stop(
      "Existen carpetas o archivos que no empiezan con 'point_', eliminar primero",
      " todos los archivos/carpetas que no esten relacionado con cloudSEN12."
    )
  }
  
  # Create the results folder
  # unlink(out_path, recursive = TRUE) 
  dir.create(sprintf("%s/highquality/manual_sc", out_path), recursive = TRUE, showWarnings = FALSE)
  
  # generate raster ref
  raster_ref <- raster(ncol=509, nrow=509, xmn=0, xmx = 5.09, ymn = 0, ymx = 5.09)
  
  # Create the results folder
  for(index in restart:length(full_points)) {
    print(sprintf("PROCESANDO: %s", basename(full_points[index])))
    print(sprintf("INDEX: %s", index))
    # get the fullpath of the npy files
    final_npy_files <- list.files(
      path = full_points[index],
      pattern = "_final\\.npy",
      full.names = TRUE,
      recursive = TRUE
    ) %>% sort()
    
    manual_npy_files <- list.files(
      path = full_points[index],
      pattern = "_user\\.npy",
      full.names = TRUE,
      recursive = TRUE
    ) %>% sort()
    
    if (length(final_npy_files) != 5 & length(manual_npy_files) != 5) {
      stop(
        sprintf(
          "El punto %s tiene %s cuando deberia tener 5. Revisar: %s",
          basename(full_points[index]),
          length(manual_npy_files),
          dirname(dirname(manual_npy_files[index]))
        )
      )
    }
    
    for (index2 in 1:5) {
      hq_manual_tif <- create_tif_hq_manual(
        final_npy_file = final_npy_files[index2],
        manual_npy_file = manual_npy_files[index2],
        raster_ref = raster_ref
      )
      
      if (length(table(getValues(hq_manual_tif))) == 1) {
        hq_manual_tif[] <- 0
      }
      
      img_id <- basename(dirname(final_npy_files[index2]))
      point_id <- basename(dirname(dirname(dirname(dirname(final_npy_files[index2])))))
      writeRaster(
        x = hq_manual_tif, 
        filename = sprintf("%s/highquality/manual_sc/%s__%s.tif", out_path, point_id, img_id),
        overwrite = TRUE
      )
    }
  }
}

scribble_manual_sc <- function(in_path, out_path, restart = 1) {
  # List all the available points in in_path
  full_points <- list.files(in_path, full.names = TRUE)
  message(sprintf("Hay %s puntos", length(full_points)))
  if (!all(grepl("point_", basename(full_points)))) {
    stop(
      "Existen carpetas o archivos que no empiezan con 'point_', eliminar primero",
      " todos los archivos/carpetas que no esten relacionado con cloudSEN12."
    )
  }
  
  
  # Create the results folder
  # unlink(out_path, recursive = TRUE)
  dir.create(sprintf("%s/scribble/manual_sc", out_path), recursive = TRUE, showWarnings = FALSE)
  
  # generate raster ref
  raster_ref <- raster(ncol=509, nrow=509, xmn=0, xmx = 5.09, ymn = 0, ymx = 5.09)
  
  # Create the results folder
  for(index in restart:length(full_points)) {
    print(sprintf("PROCESANDO: %s", basename(full_points[index])))
    print(sprintf("INDEX: %s", index))
    # get the fullpath of the npy files
    final_npy_files <- list.files(
      path = full_points[index],
      pattern = "_final\\.npy",
      full.names = TRUE,
      recursive = TRUE
    ) %>% sort()
    
    manual_npy_files <- list.files(
      path = full_points[index],
      pattern = "_user\\.npy",
      full.names = TRUE,
      recursive = TRUE
    ) %>% sort()
    
    if (length(final_npy_files) != 5 & length(manual_npy_files) != 5) {
      stop(
        sprintf(
          "El punto %s tiene %s cuando deberia tener 5. Revisar: %s",
          basename(full_points[index]),
          length(manual_npy_files),
          dirname(dirname(manual_npy_files[index]))
        )
      )
    }
    
    for (index2 in 1:5) {
      scribble_tif <- create_tif_scribble(
        final_npy_file = final_npy_files[index2],
        manual_npy_file = manual_npy_files[index2],
        raster_ref = raster_ref
      )
      
      if (length(table(getValues(scribble_tif))) == 1) {
        scribble_tif[] <- 0
      }
      
      img_id <- basename(dirname(final_npy_files[index2]))
      point_id <- basename(dirname(dirname(dirname(dirname(final_npy_files[index2])))))
      writeRaster(
        x = scribble_tif, 
        filename = sprintf("%s/scribble/manual_sc/%s__%s.tif", out_path, point_id, img_id),
        overwrite = TRUE
      )
    }
  }
}

scribble_manual_hq <- function(in_path, out_path, restart = 1) {
  # List all the available points in in_path
  full_points <- list.files(in_path, full.names = TRUE)
  message(sprintf("Hay %s puntos", length(full_points)))
  if (!all(grepl("point_", basename(full_points)))) {
    stop(
      "Existen carpetas o archivos que no empiezan con 'point_', eliminar primero",
      " todos los archivos/carpetas que no esten relacionado con cloudSEN12."
    )
  }
  
  
  # Create the results folder
  # unlink(out_path, recursive = TRUE)
  dir.create(sprintf("%s/scribble/manual_hq", out_path), recursive = TRUE, showWarnings = FALSE)
  
  # generate raster ref
  raster_ref <- raster(ncol=509, nrow=509, xmn=0, xmx = 5.09, ymn = 0, ymx = 5.09)
  raster_ref[] <- NA
  
  # Create the results folder
  for(index in restart:length(full_points)) {
    print(sprintf("PROCESANDO: %s", basename(full_points[index])))
    print(sprintf("INDEX: %s", index))
    # get the fullpath of the npy files
    final_npy_files <- list.files(
      path = full_points[index],
      pattern = "_final\\.npy",
      full.names = TRUE,
      recursive = TRUE
    ) %>% sort()
    
    for (index2 in 1:5) {
      img_id <- basename(dirname(final_npy_files[index2]))
      point_id <- basename(dirname(dirname(dirname(dirname(final_npy_files[index2])))))
      writeRaster(
        x = raster_ref, 
        filename = sprintf("%s/scribble/manual_hq/%s__%s.tif", out_path, point_id, img_id),
        overwrite = TRUE
      )
    }
  }
}

create_tif_scribble <- function(final_npy_file, manual_npy_file, raster_ref) {
  final_npy <- np$load(final_npy_file)[2:510, 2:510, 1:8]
  manual_npy <- np$load(manual_npy_file)[2:510, 2:510]
  
  clear <- manual_npy * final_npy[,,1]
  thick_c <- manual_npy * final_npy[,,2]*2
  thick_b <- manual_npy * final_npy[,,3]*3
  thin_c <- manual_npy * final_npy[,,4]*4
  thin_b <- manual_npy * final_npy[,,5]*5
  shadow_c <- manual_npy * final_npy[,,6]*6
  shadow_b <- manual_npy * final_npy[,,7]*7
  raster_ref[] <- (clear + thick_c + thick_b + thin_c + thin_b + shadow_c + shadow_b) #%>% rotate
  raster_ref[raster_ref == 0] = NA
  raster_ref - 1
}


create_tif_hq_final <- function(in_path, raster_ref) {
  png_mtx <- png::readPNG(in_path)[2:510, 2:510, ]
  rgb_img <- rgb(png_mtx[,,1], png_mtx[,,2], png_mtx[,,3])
  if (!all(unique(rgb_img) %in% c("#00FF00", "#FF0000", "#FFFF00", "#FFFFFF"))) {
    stop("El archivo no tiene colores validos. Revisar: ", in_path)  
  }
  rgb_img[rgb_img == "#FFFFFF"] = 0
  rgb_img[rgb_img == "#FFFF00"] = 1
  rgb_img[rgb_img == "#00FF00"] = 2
  rgb_img[rgb_img == "#FF0000"] = 3
  rgb_img <- as.numeric(rgb_img)
  dim(rgb_img) <- c(509, 509)
  raster_ref[] <- rgb_img
  raster_ref
}

create_tif_hq_manual <- function(final_npy_file, manual_npy_file, raster_ref) {
  final_npy <- np$load(final_npy_file)[2:510, 2:510, ]
  manual_npy <- np$load(manual_npy_file)[2:510, 2:510]
  clear <- final_npy[,,1]*1
  thick <- final_npy[,,2]*2
  thin <- final_npy[,,3]*3
  shadow <- final_npy[,,4]*4
  manual <- raster_ref
  raster_ref[] <- (clear + thick + thin + shadow) #%>% rotate
  raster_ref[raster_ref == 0] = NA
  raster_ref <- raster_ref - 1
  manual[] <- manual_npy
  final_mask <- manual*(raster_ref==0)*1 +manual*(raster_ref==1)*2 +manual*(raster_ref==2)*3 +manual*(raster_ref==3)*4
  final_mask[final_mask == 0] = NA
  final_mask - 1
}


scribble_check <- function(in_path) {
  all_points <- list.files(in_path, full.names = TRUE)
  # Checking point by point
  for (index in 1:length(all_points)) {
    datapoint <- all_points[index]
    
    if (!length(list.files(datapoint, "\\.iris$")) == 1) {
      warning(
        sprintf(
          "El punto %s tiene %s carpetas con logging. Revisar: %s", 
          basename(datapoint),
          length(list.files(datapoint, "\\.iris$")),
          datapoint
        )
      )
    }
    
    segmen_path <- sprintf("%s/%s.iris/segmentation", datapoint, basename(datapoint))
    all_seg <- list.files(segmen_path)
    if (length(all_seg) != 5) {
      warning(
        sprintf(
          "El punto %s tiene %s archivos cuando deberia tener 5. Revisar: %s", 
          basename(datapoint), length(all_seg), segmen_path
        )
      )
    }
  }
}

library(reticulate)
library(raster)
library(rgee)

source("https://gist.githubusercontent.com/csaybar/bd4f6bc45a9c4b1be190845bdb986621/raw/392164957bb6142b379b108686e19cdc4805ba2e/label_creation.R")

ee_Initialize()
np <- import("numpy")

OUTPUT_PATH <- "/home/csaybar/Downloads/data_results/" # Carpeta donde guardar los resultados.
HIGH_QUALITY_PATH <- "/home/csaybar/Downloads/data/hq/" # Carpeta con los datos high-quality.
SCRIBBLE_PATH <- "/home/csaybar/Downloads/data/scribble/" # Carpeta con los datos scribble.

# 1. Revisar si las carpetas no tienen puntos extras.
scribble_check(HIGH_QUALITY_PATH)
scribble_check(SCRIBBLE_PATH)

# CONVERTIR WARNINGS A ERRORES!
options(warn=2)

# 2. Generar los labels para los puntos con high-quality.
highquality_manual_hq(in_path = HIGH_QUALITY_PATH, out_path = OUTPUT_PATH, restart = 1)
highquality_manual_sc(HIGH_QUALITY_PATH, OUTPUT_PATH, restart = 1)

# 3. Generar los labels para los puntos con scribble.
scribble_manual_hq(SCRIBBLE_PATH, OUTPUT_PATH, restart = 1)
scribble_manual_sc(SCRIBBLE_PATH, OUTPUT_PATH, restart = 1)