Mem error while submitting job but works when ran with the R file
Shrishtee-kandoi opened this issue · comments
Script used:
Residual Expression: sQTL
sos run /restricted/projectnb/casa/xqtl-pipeline/pipeline/covariate_formatting.ipynb compute_residual \
--cwd /restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/Covariate_data_preprocessing/ \
--phenoFile /restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/leafcutter_output/batch_all_bam_intron_usage_perind.counts.gz_raw_data.qqnorm.formated.bed.gz \
--covFile /restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/Covariate_data_preprocessing/ROSMAP_xqtl_covariates_sex_death_pmi_study.ROSMAP_NIA_WGS.leftnorm.filtered.related.filtered.extracted.pca.projected.gz \
--container /restricted/projectnb/casa/xqtl-pipeline/container/singularity/bioinfo.sif \
--walltime 100h \
--numThreads 20 \
--mem 100G -J 50 -c csg.yml -q neurology
Error:
INFO: Running compute_residual:
ERROR: compute_residual (id=b3a8877311f35baf) returns an error.
ERROR: [compute_residual]: [0]: Executing script in Singularity returns an error (exitcode=-11, stderr=/restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/Covariate_data_preprocessing/batch_all_bam_intron_usage_perind.counts.gz_raw_data.qqnorm.formated.ROSMAP_xqtl_covariates_sex_death_pmi_study.ROSMAP_NIA_WGS.leftnorm.filtered.related.filtered.extracted.pca.projected.resid.bed.gz.stderr).
The script has been saved to /usr4/bf527/skandoi/.sos/8df4c2d4d05f7817//usr4/bf527/skandoi/.sos/8df4c2d4d05f7817.To reproduce the error please run:
singularity exec /restricted/projectnb/casa/xqtl-pipeline/container/singularity/bioinfo.sif Rscript /usr4/bf527/skandoi/.sos/8df4c2d4d05f7817/singularity_run_162946.R
*** caught segfault ***
address 0x7f22a4341901, cause 'invalid permissions'
R file:
/usr4/bf527/skandoi/.sos/8df4c2d4d05f7817/singularity_run_162946.R
library(dplyr)
library(readr)
pheno = read_delim('/restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/leafcutter_output/batch_all_bam_intron_usage_perind.counts.gz_raw_data.qqnorm.formated.bed.gz',delim = "\t")
covariate= read_delim('/restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/Covariate_data_preprocessing/ROSMAP_xqtl_covariates_sex_death_pmi_study.ROSMAP_NIA_WGS.leftnorm.filtered.related.filtered.extracted.pca.projected.gz',delim = "\t")
# Extract samples in both files (Outliers will be removed in here as they are no longer presented in the header of factor)
extraction_sample_list <- intersect(colnames(pheno), colnames(covariate))
if(length(extraction_sample_list) == 0){
stop("No samples are overlapped in two files!")
}
# Report identical samples:
print("Listed samples are included in the analysis:")
print(extraction_sample_list)
# Subset the data:
covariate = covariate[,extraction_sample_list]%>%as.matrix()%>%t()
pheno_id = pheno%>%select(1:4)
pheno = pheno%>%select(rownames(covariate))%>%as.matrix()%>%t()
# Get residual
pheno_resid = .lm.fit(x = cbind(1,covariate), y = pheno)$residuals
pheno_output = cbind(pheno_id, pheno_resid%>%t())
pheno_output%>%write_delim("/restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/Covariate_data_preprocessing/batch_all_bam_intron_usage_perind.counts.gz_raw_data.qqnorm.formated.ROSMAP_xqtl_covariates_sex_death_pmi_study.ROSMAP_NIA_WGS.leftnorm.filtered.related.filtered.extracted.pca.projected.resid.bed",delim = "\t")
*** caught segfault ***
address 0x7f22a4341901, cause 'invalid permissions'
Traceback:
1: vroom_(file, delim = delim %||% col_types$delim, col_names = col_names, col_types = col_types, id = id, skip = skip, col_select = col_select, name_repair = .name_repair, na = na, quote = quote, trim_ws = trim_ws, escape_double = escape_double, escape_backslash = escape_backslash, comment = comment, skip_empty_rows = skip_empty_rows, locale = locale, guess_max = guess_max, n_max = n_max, altrep = vroom_altrep(altrep), num_threads = num_threads, progress = progress)
2: vroom::vroom(file, delim = delim, col_names = col_names, col_types = col_types, col_select = { { col_select } }, id = id, .name_repair = name_repair, skip = skip, n_max = n_max, na = na, quote = quote, comment = comment, skip_empty_rows = skip_empty_rows, trim_ws = trim_ws, escape_double = escape_double, escape_backslash = escape_backslash, locale = locale, guess_max = guess_max, progress = progress, altrep = lazy, show_col_types = show_col_types, num_threads = num_threads)
3: read_delim("/restricted/projectnb/casa/skandoi/ROSMAP_DLPFC/leafcutter_output/batch_all_bam_intron_usage_perind.counts.gz_raw_data.qqnorm.formated.bed.gz", delim = "\t")
An irrecoverable exception occurred. R is aborting now ...
The problem probably have to do with the inability to read this input file due to permission error
The similar error posted online seems to be from more complicated package instead of readr
.